From cbded36b8e5c029a21cf4a510ae1e3805ac38c66 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 19 Apr 2023 01:50:05 +0000 Subject: [PATCH 01/83] Add a 'VisibilityType' flag to MRTG. Refs #439. This is a first step towards having a formal separation between the 'compiler' identity of a function and its 'type' identity, which can be less agressive and which needs to be stable during module construction. As part of this, rename 'identityHash' to 'compilerHash' and add flags to python functions to control the style of hash we want. At this point, the two hashes are the same. --- typed_python/CompilerVisibleObjectVisitor.hpp | 42 ++- typed_python/MutuallyRecursiveTypeGroup.cpp | 173 ++++++---- typed_python/MutuallyRecursiveTypeGroup.hpp | 88 ++++-- ...onSerializationContext_deserialization.cpp | 30 +- ...thonSerializationContext_serialization.cpp | 15 +- typed_python/Type.hpp | 22 +- typed_python/VisibilityType.hpp | 39 +++ typed_python/__init__.py | 2 +- typed_python/_types.cpp | 296 +++++++++++++----- .../compiler/python_to_native_converter.py | 16 +- typed_python/compiler/tests/closures_test.py | 6 +- .../compiler/tests/conversion_test.py | 4 +- .../compiler/type_wrappers/wrapper.py | 6 +- typed_python/module_representation_test.py | 4 +- typed_python/test_util.py | 2 +- typed_python/type_identity_test.py | 117 ++++--- typed_python/types_serialization_test.py | 36 +-- 17 files changed, 627 insertions(+), 271 deletions(-) create mode 100644 typed_python/VisibilityType.hpp diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index 67ae300b1..c7f467a90 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -18,6 +18,7 @@ #include "ShaHash.hpp" #include "SpecialModuleNames.hpp" +#include "VisibilityType.hpp" #include "util.hpp" #include @@ -29,6 +30,7 @@ unique hash for types and functions. ******************************/ + class VisitRecord { public: enum class kind { Hash=0, String=1, Topo=2, NameValuePair=3, Error=4 }; @@ -265,6 +267,7 @@ class CompilerVisibleObjectVisitor { template void visit( TypeOrPyobj obj, + VisibilityType visibility, const visitor_1& hashVisit, const visitor_2& nameVisit, const visitor_3& topoVisitor, @@ -273,6 +276,7 @@ class CompilerVisibleObjectVisitor { ) { visit( obj, + visibility, LambdaVisitor( hashVisit, nameVisit, topoVisitor, namedVisitor, onErr ) @@ -282,16 +286,17 @@ class CompilerVisibleObjectVisitor { template void visit( TypeOrPyobj obj, + VisibilityType visibility, const visitor_type& visitor ) { - std::vector records = recordWalk(obj); + std::vector records = recordWalk(obj, visibility); - auto it = mPastVisits.find(obj); - if (it == mPastVisits.end()) { - mPastVisits[obj] = records; + auto it = mPastVisits[visibility].find(obj); + if (it == mPastVisits[visibility].end()) { + mPastVisits[visibility][obj] = records; } else { if (it->second != records) { - checkForInstability(); + checkForInstability(visibility); throw std::runtime_error( "Found unstable object, but somehow our instability check" @@ -300,24 +305,25 @@ class CompilerVisibleObjectVisitor { } } - walk(obj, visitor); + walk(obj, visibility, visitor); } - static std::string recordWalkAsString(TypeOrPyobj obj) { + static std::string recordWalkAsString(TypeOrPyobj obj, VisibilityType visibility) { std::ostringstream s; - for (auto& record: recordWalk(obj)) { + for (auto& record: recordWalk(obj, visibility)) { s << record.toString() << "\n"; } return s.str(); } - static std::vector recordWalk(TypeOrPyobj obj) { + static std::vector recordWalk(TypeOrPyobj obj, VisibilityType visibility) { std::vector records; walk( obj, + visibility, [&](ShaHash h) { records.push_back(VisitRecord(h)); }, [&](std::string h) { records.push_back(VisitRecord(h)); }, [&](TypeOrPyobj o) { records.push_back(VisitRecord(o)); }, @@ -332,11 +338,11 @@ class CompilerVisibleObjectVisitor { mPastVisits.clear(); } - void checkForInstability() { + void checkForInstability(VisibilityType visibility) { std::vector unstable; - for (auto it = mPastVisits.begin(); it != mPastVisits.end(); ++it) { - if (it->second != recordWalk(it->first)) { + for (auto it = mPastVisits[visibility].begin(); it != mPastVisits[visibility].end(); ++it) { + if (it->second != recordWalk(it->first, visibility)) { unstable.push_back(it->first); } } @@ -352,8 +358,8 @@ class CompilerVisibleObjectVisitor { for (long k = 0; k < unstable.size() && k < 1000; k++) { s << k << " -> " << unstable[k].name() << "\n"; - std::vector linesLeft = stringifyVisitRecord(recordWalk(unstable[k])); - std::vector linesRight = stringifyVisitRecord(mPastVisits[unstable[k]]); + std::vector linesLeft = stringifyVisitRecord(recordWalk(unstable[k], visibility)); + std::vector linesRight = stringifyVisitRecord(mPastVisits[visibility][unstable[k]]); auto pad = [&](std::string s, int ct) { if (s.size() > ct) { @@ -510,6 +516,7 @@ class CompilerVisibleObjectVisitor { template static void walk( TypeOrPyobj obj, + VisibilityType visibility, const visitor_1& hashVisit, const visitor_2& nameVisit, const visitor_3& topoVisitor, @@ -517,6 +524,7 @@ class CompilerVisibleObjectVisitor { const visitor_5& onErr ) { walk(obj, + visibility, LambdaVisitor( hashVisit, nameVisit, topoVisitor, namedVisitor, onErr ) @@ -559,6 +567,7 @@ class CompilerVisibleObjectVisitor { template static void walk( TypeOrPyobj obj, + VisibilityType visibility, const visitor_type& visitor ) { PyEnsureGilAcquired getTheGil; @@ -854,5 +863,8 @@ class CompilerVisibleObjectVisitor { visitor.visitTopo((PyObject*)obj.pyobj()->ob_type); } - std::unordered_map > mPastVisits; + std::map< + VisibilityType, + std::unordered_map > + > mPastVisits; }; diff --git a/typed_python/MutuallyRecursiveTypeGroup.cpp b/typed_python/MutuallyRecursiveTypeGroup.cpp index 8396ad601..b4fdf6af9 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.cpp +++ b/typed_python/MutuallyRecursiveTypeGroup.cpp @@ -9,7 +9,7 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + WITHOUT WARRANTIES OR CONDITIONS OF ANY visibility, either express or implied. See the License for the specific language governing permissions and limitations under the License. ******************************************************************************/ @@ -18,22 +18,30 @@ #include "MutuallyRecursiveTypeGroup.hpp" -MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup() : +MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup(VisibilityType visibility) : mAnyPyObjectsIncorrectlyOrdered(false), - mIsDeserializerGroup(false) + mIsDeserializerGroup(false), + mVisibilityType(visibility) { } -MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup(ShaHash hash) : +MutuallyRecursiveTypeGroup::MutuallyRecursiveTypeGroup( + ShaHash hash, + VisibilityType visibility +) : mAnyPyObjectsIncorrectlyOrdered(false), mIntendedHash(hash), + mVisibilityType(visibility), mIsDeserializerGroup(true) { } -MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::DeserializerGroup(ShaHash hash) { - return new MutuallyRecursiveTypeGroup(hash); +MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::DeserializerGroup( + ShaHash hash, + VisibilityType k +) { + return new MutuallyRecursiveTypeGroup(hash, k); } @@ -77,11 +85,13 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { // and then copy them in MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( - mIndexToObject.begin()->second + mIndexToObject.begin()->second, + mVisibilityType ); MutuallyRecursiveTypeGroup* canonical = MutuallyRecursiveTypeGroup::groupAndIndexFor( - mIndexToObject.begin()->second + mIndexToObject.begin()->second, + mVisibilityType ).first; if (mIndexToObject.size() != canonical->mIndexToObject.size()) { @@ -118,8 +128,8 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { std::lock_guard lock(mMutex); // if this intended hash doesn't match anything, just copy it in - if (mIntendedHashToGroup.find(mIntendedHash) == mIntendedHashToGroup.end()) { - mIntendedHashToGroup[mIntendedHash] = this; + if (mIntendedHashToGroup[mVisibilityType].find(mIntendedHash) == mIntendedHashToGroup[mVisibilityType].end()) { + mIntendedHashToGroup[mVisibilityType][mIntendedHash] = this; } } } @@ -135,22 +145,22 @@ void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { std::lock_guard lock(mMutex); // see if this official hash has been seen yet - if (mHashToGroup.find(mHash) == mHashToGroup.end()) { + if (mHashToGroup[mVisibilityType].find(mHash) == mHashToGroup[mVisibilityType].end()) { // this is the official MRTG for this hash, so we install ourselves // as the official group and register ourselves in the various type memos. // this group will come up in the reverse lookup now. - mHashToGroup[mHash] = this; + mHashToGroup[mVisibilityType][mHash] = this; } for (auto& typeAndOrder: mObjectToIndex) { - auto it = mTypeGroups.find(typeAndOrder.first); + auto it = mTypeGroups[mVisibilityType].find(typeAndOrder.first); - if (it == mTypeGroups.end()) { + if (it == mTypeGroups[mVisibilityType].end()) { // this type has never been installed - go ahead and add it. It's possible // there are multiple copies of this type in the system - mTypeGroups[typeAndOrder.first] = std::make_pair(this, typeAndOrder.second); + mTypeGroups[mVisibilityType][typeAndOrder.first] = std::make_pair(this, typeAndOrder.second); - if (typeAndOrder.first.type()) { + if (typeAndOrder.first.type() && mVisibilityType == VisibilityType::Identity) { Type* t = typeAndOrder.first.type(); t->setRecursiveTypeGroup(this, typeAndOrder.second); } @@ -161,9 +171,14 @@ void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { } //static -void MutuallyRecursiveTypeGroup::visibleFrom(TypeOrPyobj root, std::vector& outReachable) { +void MutuallyRecursiveTypeGroup::visibleFrom( + TypeOrPyobj root, + std::vector& outReachable, + VisibilityType visibility +) { CompilerVisibleObjectVisitor::singleton().visit( root, + visibility, [&](ShaHash h) {}, [&](const std::string& s) {}, [&](TypeOrPyobj t) { outReachable.push_back(t); }, @@ -188,16 +203,16 @@ void MutuallyRecursiveTypeGroup::setIndexToObject(int32_t index, TypeOrPyobj obj } } -bool MutuallyRecursiveTypeGroup::objectIsUnassigned(TypeOrPyobj obj) { +bool MutuallyRecursiveTypeGroup::objectIsUnassigned(TypeOrPyobj obj, VisibilityType visibility) { // we can check if we're installed without hitting the lock - if (obj.type() && obj.type()->hasTypeGroup()) { + if (visibility == VisibilityType::Identity && obj.type() && obj.type()->hasTypeGroup()) { return false; } std::lock_guard lock(mMutex); // if we've already installed this group into 'mTypeGroups' - if (mTypeGroups.find(obj) != mTypeGroups.end()) { + if (mTypeGroups[visibility].find(obj) != mTypeGroups[visibility].end()) { return false; } @@ -243,20 +258,22 @@ std::string MutuallyRecursiveTypeGroup::repr(bool deep) { s << levelPrefix << " " << ix << " -> " << obj.name() << "\n"; if (!deep) { - s << CompilerVisibleObjectVisitor::recordWalkAsString(obj) << "\n"; + s << CompilerVisibleObjectVisitor::recordWalkAsString( + obj, mVisibilityType + ) << "\n"; } if (deep) { std::vector visible; - visibleFrom(obj, visible); + visibleFrom(obj, visible, mVisibilityType); for (auto v: visible) { MutuallyRecursiveTypeGroup* subgroup; int ixInSubgroup; - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(v); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(v, mVisibilityType); std::tie(subgroup, ixInSubgroup) = - MutuallyRecursiveTypeGroup::groupAndIndexFor(v); + MutuallyRecursiveTypeGroup::groupAndIndexFor(v, mVisibilityType); if (subgroup) { if (seen.find(subgroup) == seen.end()) { @@ -287,7 +304,10 @@ std::string MutuallyRecursiveTypeGroup::repr(bool deep) { // were unable to uniquely pick it. // static -std::pair MutuallyRecursiveTypeGroup::computeRoot(const std::set& topos) { +std::pair MutuallyRecursiveTypeGroup::computeRoot( + const std::set& topos, + VisibilityType visibility +) { // we look at python objects in this function, so we need to be holding the GIL PyEnsureGilAcquired getTheGil; @@ -387,13 +407,14 @@ std::pair MutuallyRecursiveTypeGroup::computeRoot(const std:: CompilerVisibleObjectVisitor::singleton().visit( t, + visibility, [&](ShaHash h) { newHash += h; }, [&](const std::string& s) { newHash += ShaHash(s); }, [&](TypeOrPyobj t) { if (curHashes.find(t) != curHashes.end()) { newHash += curHashes[t]; } else { - newHash += shaHash(t); + newHash += shaHash(t, visibility); } }, [&](const std::string& s, TypeOrPyobj t) { @@ -402,7 +423,7 @@ std::pair MutuallyRecursiveTypeGroup::computeRoot(const std:: if (curHashes.find(t) != curHashes.end()) { newHash += curHashes[t]; } else { - newHash += shaHash(t); + newHash += shaHash(t, visibility); } }, [&](const std::string& err) {} @@ -435,7 +456,10 @@ std::pair MutuallyRecursiveTypeGroup::computeRoot(const std:: } // static -void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(const std::set& topos) { +void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup( + const std::set& topos, + VisibilityType visibility +) { // check to see if any of our topos has been installed in a type group already. If so, // then they ALL should have been installed in a type group. This can happen if two threads // are computing type groups at the same time: imagine a cycle of many objects - both @@ -447,7 +471,7 @@ void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(const std::setmAnyPyObjectsIncorrectlyOrdered) = computeRoot(topos); + std::tie(root, group->mAnyPyObjectsIncorrectlyOrdered) = computeRoot(topos, visibility); std::map ordering; @@ -493,6 +517,7 @@ void MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(const std::set lock(mMutex); - auto it = mHashToGroup.find(hash); - if (it == mHashToGroup.end()) { + auto it = mHashToGroup[visibility].find(hash); + if (it == mHashToGroup[visibility].end()) { return nullptr; } return it->second; } -MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::getGroupFromIntendedHash(ShaHash hash) { +MutuallyRecursiveTypeGroup* MutuallyRecursiveTypeGroup::getGroupFromIntendedHash(ShaHash hash, VisibilityType visibility) { std::lock_guard lock(mMutex); - auto it = mIntendedHashToGroup.find(hash); - if (it != mIntendedHashToGroup.end()) { + auto it = mIntendedHashToGroup[visibility].find(hash); + if (it != mIntendedHashToGroup[visibility].end()) { return it->second; } - auto it2 = mHashToGroup.find(hash); - if (it2 != mHashToGroup.end()) { + auto it2 = mHashToGroup[visibility].find(hash); + if (it2 != mHashToGroup[visibility].end()) { return it2->second; } @@ -555,7 +580,7 @@ void MutuallyRecursiveTypeGroup::computeHash() { thread_local static MutuallyRecursiveTypeGroup* currentlyHashing = nullptr; if (currentlyHashing) { - CompilerVisibleObjectVisitor::singleton().checkForInstability(); + CompilerVisibleObjectVisitor::singleton().checkForInstability(mVisibilityType); std::ostringstream errMsg; @@ -615,17 +640,20 @@ ShaHash MutuallyRecursiveTypeGroup::computeTopoHash(TypeOrPyobj toHash) { return; } - auto groupAndIx = groupAndIndexFor(o); + auto groupAndIx = groupAndIndexFor(o, mVisibilityType); if (!groupAndIx.first || groupAndIx.second == -1) { - CompilerVisibleObjectVisitor::singleton().checkForInstability(); + CompilerVisibleObjectVisitor::singleton().checkForInstability(mVisibilityType); throw std::runtime_error( "All reachable objects should be in this group or hashed already. " "Instead, this object is not hashed and not local: " + o.name() + (groupAndIx.first ? "\n\nhas a group but not an index\n\n":"") + - + "\n\nwithin\n\n" + toHash.name() + "\n\nwalk is\n" + CompilerVisibleObjectVisitor::recordWalkAsString(toHash) - + "\n\nwalk of not hashed is\n" + CompilerVisibleObjectVisitor::recordWalkAsString(o) + + "\n\nwithin\n\n" + toHash.name() + "\n\nwalk is\n" + + CompilerVisibleObjectVisitor::recordWalkAsString(toHash, mVisibilityType) + + "\n\nwalk of not hashed is\n" + + CompilerVisibleObjectVisitor::recordWalkAsString(o, mVisibilityType) + + "\nVT = " + visibilityTypeToStr(mVisibilityType) ); } @@ -635,6 +663,7 @@ ShaHash MutuallyRecursiveTypeGroup::computeTopoHash(TypeOrPyobj toHash) { // walk one layer deep into our objects CompilerVisibleObjectVisitor::singleton().visit( toHash, + mVisibilityType, [&](ShaHash h) { res += ShaHash(1) + h; }, [&](std::string o) { res += ShaHash(2) + ShaHash(o); }, [&](TypeOrPyobj o) { @@ -654,7 +683,9 @@ ShaHash MutuallyRecursiveTypeGroup::computeTopoHash(TypeOrPyobj toHash) { // compiler is concerned) class MutuallyRecursiveTypeGroupSearch { public: - MutuallyRecursiveTypeGroupSearch() {} + MutuallyRecursiveTypeGroupSearch(VisibilityType visibility) : + mVisibilityType(visibility) + {} // this object is new and needs its own group void pushGroup(TypeOrPyobj o) { @@ -681,15 +712,16 @@ class MutuallyRecursiveTypeGroupSearch { // now recurse into the subtypes CompilerVisibleObjectVisitor::singleton().visit( o, + mVisibilityType, [&](ShaHash h) {}, [&](std::string o) {}, [&](TypeOrPyobj o) { - if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o)) { + if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o, mVisibilityType)) { mGroupOutboundEdges.back()->push_back(o); } }, [&](std::string name, TypeOrPyobj o) { - if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o)) { + if (MutuallyRecursiveTypeGroup::objectIsUnassigned(o, mVisibilityType)) { mGroupOutboundEdges.back()->push_back(o); } }, @@ -706,7 +738,7 @@ class MutuallyRecursiveTypeGroupSearch { void doOneStep() { if (mGroupOutboundEdges.back()->size() == 0) { // this group is finished.... - MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(*mGroups.back()); + MutuallyRecursiveTypeGroup::buildCompilerRecursiveGroup(*mGroups.back(), mVisibilityType); for (auto gElt: *mGroups.back()) { mInAGroup.erase(gElt); @@ -717,9 +749,10 @@ class MutuallyRecursiveTypeGroupSearch { } else { // pop an outbound edge and see where does it go? TypeOrPyobj o = mGroupOutboundEdges.back()->back(); + mGroupOutboundEdges.back()->pop_back(); - if (!MutuallyRecursiveTypeGroup::objectIsUnassigned(o)) { + if (!MutuallyRecursiveTypeGroup::objectIsUnassigned(o, mVisibilityType)) { return; } @@ -760,14 +793,16 @@ class MutuallyRecursiveTypeGroupSearch { // for each group, the set of things it reaches out to std::vector > > mGroupOutboundEdges; + + VisibilityType mVisibilityType; }; // static -void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root) { +void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root, VisibilityType visibility) { // we do things with pyobj refcounts, so we need to hold the gil. PyEnsureGilAcquired getTheGil; - MutuallyRecursiveTypeGroupSearch groupFinder; + MutuallyRecursiveTypeGroupSearch groupFinder(visibility); static thread_local int count = 0; count++; @@ -788,12 +823,12 @@ void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root) { count--; } -ShaHash MutuallyRecursiveTypeGroup::shaHash(TypeOrPyobj o) { +ShaHash MutuallyRecursiveTypeGroup::shaHash(TypeOrPyobj o, VisibilityType visibility) { if (o.pyobj() && CompilerVisibleObjectVisitor::isSimpleConstant(o.pyobj())) { return shaHashOfSimplePyObjectConstant(o.pyobj()); } - auto groupAndIx = MutuallyRecursiveTypeGroup::groupAndIndexFor(o); + auto groupAndIx = MutuallyRecursiveTypeGroup::groupAndIndexFor(o, visibility); if (!groupAndIx.first) { return ShaHash(); @@ -803,16 +838,21 @@ ShaHash MutuallyRecursiveTypeGroup::shaHash(TypeOrPyobj o) { } // static -std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor(PyObject* o) { - return groupAndIndexFor(TypeOrPyobj::withoutIntern(o)); +std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor( + PyObject* o, VisibilityType visibility +) { + return groupAndIndexFor(TypeOrPyobj::withoutIntern(o), visibility); } // static -std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor(TypeOrPyobj o) { +std::pair MutuallyRecursiveTypeGroup::groupAndIndexFor( + TypeOrPyobj o, + VisibilityType visibility +) { std::lock_guard lock(mMutex); - auto it = mTypeGroups.find(o); - if (it != mTypeGroups.end()) { + auto it = mTypeGroups[visibility].find(o); + if (it != mTypeGroups[visibility].end()) { return std::pair( it->second.first, it->second.first->indexOfObjectInThisGroup(o) ); @@ -1004,13 +1044,22 @@ ShaHash MutuallyRecursiveTypeGroup::shaHashOfSimplePyObjectConstant(PyObject* h) } //static -std::unordered_map > MutuallyRecursiveTypeGroup::mTypeGroups; +std::map< + VisibilityType, + std::unordered_map > +> MutuallyRecursiveTypeGroup::mTypeGroups; //static -std::map MutuallyRecursiveTypeGroup::mHashToGroup; +std::map< + VisibilityType, + std::map +> MutuallyRecursiveTypeGroup::mHashToGroup; //static -std::map MutuallyRecursiveTypeGroup::mIntendedHashToGroup; +std::map< + VisibilityType, + std::map +> MutuallyRecursiveTypeGroup::mIntendedHashToGroup; //static std::recursive_mutex MutuallyRecursiveTypeGroup::mMutex; diff --git a/typed_python/MutuallyRecursiveTypeGroup.hpp b/typed_python/MutuallyRecursiveTypeGroup.hpp index ec0af210b..8ea020c80 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.hpp +++ b/typed_python/MutuallyRecursiveTypeGroup.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2020 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,17 +18,47 @@ #include "TypeOrPyobj.hpp" #include "ShaHash.hpp" +#include "VisibilityType.hpp" + #include #include class MutuallyRecursiveTypeGroupSearch; +/***** +MutuallyRecursiveTypeGroup (or MRTG) provides infrstructure for hashing groups of python +type and module objects. The primary use cases for this infrastructure are for identifying +truly identical Type objects that may have been constructed separately (either via deserialization +or directly via the API) and for hashing Type and function objects so that the compiler cache +can formally identify that a function will compile to the same code. + +We have multiple angles at which we can look at an object. The simplest is 'identity', where +we wish to associate a hash with an object so that we are guaranteed that two objects +with the same hash will behave identically (not including pointer identity). For instance, +two instances of the same function object visible at module-level scope in a globally visible +module would be 'identical', even if the module object changes. + +The more stringent hash is the 'compiler' hash, in which two objects that have the same +hash would look identical to the compiler now and at all points in the future. In this model, +we need to deeply walk into the object in the same manner the compiler would. For this to +work, we make the same assumption that we make in the main compiler: once we have finished +the module import process, all of the objects defined in the module are fully defined. +In particular this means that the identify of module members will not chnage (essentially, +that the module's dict is immutable), and that class members of classes defined in that module +will not change (the class dicts are immutable). Mutable objects such as module-level lists +may change during runtime, and so we don't consider them part of the compiler hash. + +In order to hash python objects, we have to find groups of mutually recursive objects. As +an example, if 'f' and 'g' are mutually recursive functions, then if you change either of +their code objects, then both of their compiler hashes should change. To this end, we define +the MutuallyRecursiveTypeGroup (MRTG), which builds the DAG of object cycles. The MRTGs +are defined either as Identity or Compiler groups, depending on the use case. Objects +found in MRTGs are memoized in the program and won't ever be released, so be careful what +you hash. + +******/ -// represents a group of Type* and PyObject* instances that can 'see' each other according -// to the compiler. These instances need to be serialized and hashed as a group. The hash -// of the instances depends on the hash of the group, and the hash of the group depends on -// the set of instances contained inside of it. class MutuallyRecursiveTypeGroup { public: //return an empty 'builder' group. Builder groups can be built up and then @@ -41,7 +71,7 @@ class MutuallyRecursiveTypeGroup { //Regardless, you're guaranteed that after you finalize this group, there will be only //one copy of the types or singleton objects it contains. The types you put in may be //discarded, so don't keep a reference to them. - static MutuallyRecursiveTypeGroup* DeserializerGroup(ShaHash intendedHash); + static MutuallyRecursiveTypeGroup* DeserializerGroup(ShaHash intendedHash, VisibilityType visibility); bool isDeserializerGroup() const { return mIsDeserializerGroup; @@ -61,6 +91,10 @@ class MutuallyRecursiveTypeGroup { return mHash; } + VisibilityType visibilityType() const { + return mVisibilityType; + } + const std::map& getIndexToObject() const { return mIndexToObject; } @@ -68,44 +102,44 @@ class MutuallyRecursiveTypeGroup { std::string repr(bool deep=false); // if we have an official MRTG for this hash, return it. - static MutuallyRecursiveTypeGroup* getGroupFromHash(ShaHash h); + static MutuallyRecursiveTypeGroup* getGroupFromHash(ShaHash h, VisibilityType k); // if we have an MRTG with this 'intended' hash, return that. - static MutuallyRecursiveTypeGroup* getGroupFromIntendedHash(ShaHash h); + static MutuallyRecursiveTypeGroup* getGroupFromIntendedHash(ShaHash h, VisibilityType k); // construct an MRTG on this. After this call, this object will be in the type memo // and calls to 'groupAndIndexFor' and shaHash will succeed - static void ensureRecursiveTypeGroup(TypeOrPyobj root); + static void ensureRecursiveTypeGroup(TypeOrPyobj root, VisibilityType k); // return the current group head, assuming one has been created. If you haven't called // ensureRecursiveTypeGroup, you may get a nullptr for the group. Note that creating // a TypeOrPyobj will leak a reference to 'o', so don't do it on any old object. - static std::pair groupAndIndexFor(TypeOrPyobj o); + static std::pair groupAndIndexFor(TypeOrPyobj o, VisibilityType k); // lookup an object without increffing it - static std::pair groupAndIndexFor(PyObject* o); + static std::pair groupAndIndexFor(PyObject* o, VisibilityType k); // return the current sha-hash for this object, or ShaHash() if its not in a type // group yet. - static ShaHash shaHash(TypeOrPyobj t); + static ShaHash shaHash(TypeOrPyobj t, VisibilityType k); // ensure this object is in a type group and then return the set of reachable instances - static void visibleFrom(TypeOrPyobj root, std::vector& outReachable); + static void visibleFrom(TypeOrPyobj root, std::vector& outReachable, VisibilityType k); private: - MutuallyRecursiveTypeGroup(); + MutuallyRecursiveTypeGroup(VisibilityType visibility); - MutuallyRecursiveTypeGroup(ShaHash hash); + MutuallyRecursiveTypeGroup(ShaHash hash, VisibilityType visibility); static ShaHash shaHashOfSimplePyObjectConstant(PyObject* h); - static bool objectIsUnassigned(TypeOrPyobj obj); + static bool objectIsUnassigned(TypeOrPyobj obj, VisibilityType visibility); ShaHash computeTopoHash(TypeOrPyobj t); void _computeHashAndInstall(); - static void buildCompilerRecursiveGroup(const std::set& types); + static void buildCompilerRecursiveGroup(const std::set& types, VisibilityType visibility); static void installTypeHash(Type* t); @@ -113,7 +147,10 @@ class MutuallyRecursiveTypeGroup { // given a collection of TypeOrPyobj, figure out which one is the 'root' for the group // this must be stable regardless of the ordering. - static std::pair computeRoot(const std::set& types); + static std::pair computeRoot( + const std::set& types, + VisibilityType visibility + ); // static mutex that guards all the relevant data structures. Changes to all static // structures below must be made while holding the mutex. You may not hold the mutex @@ -121,11 +158,17 @@ class MutuallyRecursiveTypeGroup { static std::recursive_mutex mMutex; // for each python object that's in a type group, the group and index within the group - static std::unordered_map > mTypeGroups; + static std::map< + VisibilityType, + std::unordered_map > + > mTypeGroups; - static std::map mIntendedHashToGroup; + static std::map< + VisibilityType, + std::map + > mIntendedHashToGroup; - static std::map mHashToGroup; + static std::map > mHashToGroup; void computeHash(); @@ -133,6 +176,9 @@ class MutuallyRecursiveTypeGroup { // if not, we can't modify it and it may or may not be officially installed. bool mIsDeserializerGroup; + // what kind of group are we + VisibilityType mVisibilityType; + // the sha hash of this group within this codebase ShaHash mHash; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index fce5b8c3d..2bc9f8efb 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -613,7 +613,10 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur groupHash = ShaHash::fromDigest(b.readStringObject()); hasGroupHash = true; - outGroup = MutuallyRecursiveTypeGroup::getGroupFromIntendedHash(groupHash); + outGroup = MutuallyRecursiveTypeGroup::getGroupFromIntendedHash( + groupHash, + VisibilityType::Identity + ); if (memo != -1 && outGroup) { b.addCachedPointer(memo, (void*)outGroup, nullptr); @@ -625,7 +628,10 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } if (!outGroup) { - outGroup = MutuallyRecursiveTypeGroup::DeserializerGroup(groupHash); + outGroup = MutuallyRecursiveTypeGroup::DeserializerGroup( + groupHash, + VisibilityType::Identity + ); if (memo != -1) { b.addCachedPointer(memo, (void*)outGroup, nullptr); @@ -1009,10 +1015,16 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur ); } - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)namedObj); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)namedObj, + VisibilityType::Identity + ); MutuallyRecursiveTypeGroup* group = ( - MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)namedObj).first + MutuallyRecursiveTypeGroup::groupAndIndexFor( + (PyObject*)namedObj, + VisibilityType::Identity + ).first ); if (!outGroup) { @@ -1047,10 +1059,16 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur throw PythonExceptionSet(); } - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)instance); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)instance, + VisibilityType::Identity + ); MutuallyRecursiveTypeGroup* group = ( - MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)instance).first + MutuallyRecursiveTypeGroup::groupAndIndexFor( + (PyObject*)instance, + VisibilityType::Identity + ).first ); if (!outGroup) { diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 34282de78..ee78db335 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -41,7 +41,10 @@ void PythonSerializationContext::serializePythonObject(PyObject* o, Serializatio // because otherwise we'd create a dependency during serialization on _whether_ the // group was already created, which would introduce a nondeterminism into serilization // based on the compiler state - auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor(o); + auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor( + o, + VisibilityType::Identity + ); // if we have a group, check whether we've already memoized this group in the stream // this prevents us from initiating the serialization of a new group here. @@ -248,8 +251,14 @@ void PythonSerializationContext::serializePythonObjectByNameOrRepresentation(PyO if (PyType_Check(o)) { // if its a Type object, this is the place to put it into a mutually recursive type // group - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(o); - auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor(o); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + o, + VisibilityType::Identity + ); + auto groupAndIndex = MutuallyRecursiveTypeGroup::groupAndIndexFor( + o, + VisibilityType::Identity + ); b.writeBeginCompound(FieldNumbers::RECURSIVE_OBJECT); serializeMutuallyRecursiveTypeGroup(groupAndIndex.first, b, 0); diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index cb62247b5..a99145e6b 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -611,7 +611,7 @@ class Type { return false; } - return t1->identityHash() == t2->identityHash(); + return t1->compilerHash() == t2->compilerHash(); } //this checks _strict_ subclass. X is not a subclass of itself. @@ -842,7 +842,10 @@ class Type { *****/ MutuallyRecursiveTypeGroup* getRecursiveTypeGroup() { if (!mTypeGroup) { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(this); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + this, + VisibilityType::Identity + ); if (!mTypeGroup) { throw std::runtime_error("Somehow, this type is not in a type group!"); @@ -877,18 +880,27 @@ class Type { type that's in our group. This gives us a unique signature for the group. The final hash is then that hash plus our id within the group. ******/ - ShaHash identityHash() { + ShaHash compilerHash() { // if we've never initialized our hash if (mIdentityHash == ShaHash()) { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(this); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + this, + VisibilityType::Identity + ); - mIdentityHash = MutuallyRecursiveTypeGroup::shaHash(this); + mIdentityHash = MutuallyRecursiveTypeGroup::shaHash( + this, + VisibilityType::Identity + ); } return mIdentityHash; } void setRecursiveTypeGroup(MutuallyRecursiveTypeGroup* group, int32_t index) { + if (group->visibilityType() != VisibilityType::Identity) { + throw std::runtime_error("A Type's MutuallyRecursiveTypeGroup needs to be an Identity group"); + } mTypeGroup = group; mRecursiveTypeGroupIndex = index; } diff --git a/typed_python/VisibilityType.hpp b/typed_python/VisibilityType.hpp new file mode 100644 index 000000000..dbcaa8b0c --- /dev/null +++ b/typed_python/VisibilityType.hpp @@ -0,0 +1,39 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + + +enum class VisibilityType { + // compute the identity of the object as far as the interpreter is concerned, + // excluding any notion of pointer identity. This is used to memoize Type instances. + // We don't look into module-level members when computing this value, so this is valid + // as soon as a type object is constructed + Identity = 1, + // compute the identity of the object as far as the compiler itself is concerned. + // this is stronger than the notion of the Identity hash because the + Compiler = 2, +}; + +inline std::string visibilityTypeToStr(VisibilityType v) { + if (v == VisibilityType::Identity) { + return "Identity"; + } + if (v == VisibilityType::Compiler) { + return "Compiler"; + } + return "Unknown"; +} diff --git a/typed_python/__init__.py b/typed_python/__init__.py index ca8495770..3a1b0ea18 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -48,7 +48,7 @@ Alternative, Value, serialize, deserialize, serializeStream, deserializeStream, PointerTo, RefTo, Dict, validateSerializedObject, validateSerializedObjectStream, decodeSerializedObject, getOrSetTypeResolver, Set, Class, Type, BoundMethod, - TypedCell, pointerTo, refTo, copy, identityHash, PythonObjectOfType, + TypedCell, pointerTo, refTo, copy, compilerHash, PythonObjectOfType, deepBytecount, deepcopy, deepcopyContiguous, totalBytesAllocatedInSlabs, deepBytecountAndSlabs, Slab, totalBytesAllocatedOnFreeStore, diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index c2b52dfb6..875d79957 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2057,6 +2057,54 @@ PyObject *createPyCell(PyObject* nullValue, PyObject* args) { return PyCell_New(PyTuple_GetItem(args, 0)); } +PyDoc_STRVAR( + compilerHash_doc, + "compilerHash(T) -> str\n\n" + "Compute the 'compiler hash' of a type or function T.\n\n" + "The 'compilerHash' of a function or Type is a sha-hash of all objects visible to the\n" + "compiler starting from 'T'. If two python objects have the same compiler hash,\n" + "then the compiler will produce identical code for them.\n\n" + "The compilerHash is computed by walking into a type or function and looking at all\n" + "the objects that the compiler would be willing to look at. For instance, if 'f' calls\n" + "a function 'g' and 'g' is visibile to 'f' through its module dictionary, the compiler\n" + "will know to compile and call 'f' and so 'f' will be included in this walk. In contrast,\n" + "if 'f' accesses a list object through its closure, the contents of that list will not be\n" + "contained in the cache since the contents of the list could change.\n" +); +PyObject *compilerHash(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "compilerHash takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + ShaHash hash; + + return translateExceptionToPyObject([&]() { + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)a1, + VisibilityType::Compiler + ); + hash = MutuallyRecursiveTypeGroup::shaHash( + (PyObject*)a1, + VisibilityType::Compiler + ); + + return PyBytes_FromStringAndSize((const char*)&hash, sizeof(ShaHash)); + }); +} + +PyDoc_STRVAR( + identityHash_doc, + "identityHash(T) -> str\n\n" + "Compute the 'identity hash' of a type or function T.\n\n" + "The 'identityHash' of a function or Type is a sha-hash of all non-mutable content\n" + "reachable from the object. If two Type objects have the same identityHash then they\n" + "will in fact be identical instances.\n\n" + "The identityHash is computed similarly to the compilerHash, except that we don't look\n" + "inside of modules. This allows our identity hash to be stable even as a module\n" + "is being defined." +); PyObject *identityHash(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "identityHash takes 1 positional argument"); @@ -2067,8 +2115,14 @@ PyObject *identityHash(PyObject* nullValue, PyObject* args) { ShaHash hash; return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - hash = MutuallyRecursiveTypeGroup::shaHash((PyObject*)a1); + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( + (PyObject*)a1, + VisibilityType::Identity + ); + hash = MutuallyRecursiveTypeGroup::shaHash( + (PyObject*)a1, + VisibilityType::Identity + ); return PyBytes_FromStringAndSize((const char*)&hash, sizeof(ShaHash)); }); @@ -2682,18 +2736,28 @@ PyObject *touchCompiledSpecializations(PyObject* nullValue, PyObject* args) { return incref(Py_None); } -PyObject *isRecursive(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "isRecursive takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *isRecursive(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + MutuallyRecursiveTypeGroup* group = MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; if (!group) { return incref(Py_False); @@ -2731,10 +2795,27 @@ PyDoc_STRVAR( "errors while computing hashes because the assumptions of the hasher are violated." ); -PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args) { +PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { + const char* visibility = "compiler"; + static const char *kwlist[] = {"visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|s", (char**)kwlist, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + try { - CompilerVisibleObjectVisitor::singleton().checkForInstability(); + CompilerVisibleObjectVisitor::singleton().checkForInstability(vis); return incref(Py_None); } catch(std::runtime_error& err) { @@ -2743,33 +2824,59 @@ PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args) { }); } -PyObject *typeWalkRecord(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "typeWalkRecord takes 1 positional argument"); - return NULL; +PyObject *typeWalkRecord(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); return translateExceptionToPyObject([&]() { - TypeOrPyobj obj((PyObject*)a1); + TypeOrPyobj obj(instance); return PyUnicode_FromString( - ("Walk of " + obj.name() + ":\n\n" + CompilerVisibleObjectVisitor::recordWalkAsString(obj)).c_str() + ("Walk of " + obj.name() + ":\n\n" + + CompilerVisibleObjectVisitor::recordWalkAsString(obj, vis) + ).c_str() ); }); } -PyObject *typesAndObjectsVisibleToCompilerFrom(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "typesAndObjectsVisibleToCompilerFrom takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - +PyObject *typesAndObjectsVisibleToCompilerFrom(PyObject* nullValue, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + std::vector visible; - MutuallyRecursiveTypeGroup::visibleFrom((PyObject*)a1, visible); + MutuallyRecursiveTypeGroup::visibleFrom(instance, visible, vis); PyObjectStealer res(PyList_New(0)); @@ -2786,26 +2893,39 @@ PyObject *typesAndObjectsVisibleToCompilerFrom(PyObject* nullValue, PyObject* ar }); } -PyObject *recursiveTypeGroupHash(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "recursiveTypeGroupHash takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *recursiveTypeGroupHash(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } + + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + MutuallyRecursiveTypeGroup* group = nullptr; + + if (PyType_Check(instance) && vis == VisibilityType::Identity) { + Type* typeArg = PyInstance::extractTypeFrom((PyTypeObject*)instance); - return translateExceptionToPyObject([&]() { - if (PyType_Check(a1)) { - Type* typeArg = PyInstance::extractTypeFrom(a1); if (typeArg) { group = typeArg->getRecursiveTypeGroup(); } } if (!group) { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + group = MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; } if (!group) { @@ -2816,18 +2936,30 @@ PyObject *recursiveTypeGroupHash(PyObject* nullValue, PyObject* args) { }); } -PyObject *recursiveTypeGroupReprDeepFlag(PyObject* nullValue, PyObject* args, bool deep) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "recursiveTypeGroupRepr takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *recursiveTypeGroupReprDeepFlag(PyObject* nullValue, PyObject* args, PyObject* kwargs, bool deep) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + MutuallyRecursiveTypeGroup* group = nullptr; + + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + group = MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; if (!group) { return incref(Py_None); @@ -2837,28 +2969,43 @@ PyObject *recursiveTypeGroupReprDeepFlag(PyObject* nullValue, PyObject* args, bo }); } -PyObject *recursiveTypeGroupRepr(PyObject* nullValue, PyObject* args) { - return recursiveTypeGroupReprDeepFlag(nullValue, args, false); +PyObject *recursiveTypeGroupRepr(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return recursiveTypeGroupReprDeepFlag(nullValue, args, kwargs, false); } -PyObject *recursiveTypeGroupDeepRepr(PyObject* nullValue, PyObject* args) { - return recursiveTypeGroupReprDeepFlag(nullValue, args, true); +PyObject *recursiveTypeGroupDeepRepr(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return recursiveTypeGroupReprDeepFlag(nullValue, args, kwargs, true); } -PyObject *recursiveTypeGroup(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "recursiveTypeGroup takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *recursiveTypeGroup(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + int createIfNotExist = 1; + static const char *kwlist[] = {"instance", "visibility", "createIfNotExist", NULL}; - MutuallyRecursiveTypeGroup* group = nullptr; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|sp", (char**)kwlist, &instance, &visibility, &createIfNotExist)) { + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]() { - MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup((PyObject*)a1); - group = MutuallyRecursiveTypeGroup::groupAndIndexFor((PyObject*)a1).first; + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } + + if (createIfNotExist) { + MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(instance, vis); + } + + MutuallyRecursiveTypeGroup* group = + MutuallyRecursiveTypeGroup::groupAndIndexFor(instance, vis).first; if (!group) { return incref(Py_None); @@ -3289,7 +3436,8 @@ static PyMethodDef module_methods[] = { {"decodeSerializedObject", (PyCFunction)decodeSerializedObject, METH_VARARGS, NULL}, {"validateSerializedObject", (PyCFunction)validateSerializedObject, METH_VARARGS, NULL}, {"validateSerializedObjectStream", (PyCFunction)validateSerializedObjectStream, METH_VARARGS, NULL}, - {"identityHash", (PyCFunction)identityHash, METH_VARARGS, NULL}, + {"compilerHash", (PyCFunction)compilerHash, METH_VARARGS, compilerHash_doc}, + {"identityHash", (PyCFunction)identityHash, METH_VARARGS, identityHash_doc}, {"serializeStream", (PyCFunction)serializeStream, METH_VARARGS, NULL}, {"deserializeStream", (PyCFunction)deserializeStream, METH_VARARGS, NULL}, {"is_default_constructible", (PyCFunction)is_default_constructible, METH_VARARGS, NULL}, @@ -3301,15 +3449,15 @@ static PyMethodDef module_methods[] = { {"isBinaryCompatible", (PyCFunction)isBinaryCompatible, METH_VARARGS, NULL}, {"Forward", (PyCFunction)MakeForward, METH_VARARGS, NULL}, {"allForwardTypesResolved", (PyCFunction)allForwardTypesResolved, METH_VARARGS, NULL}, - {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS, NULL}, - {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS, NULL}, - {"recursiveTypeGroupDeepRepr", (PyCFunction)recursiveTypeGroupDeepRepr, METH_VARARGS, NULL}, - {"recursiveTypeGroupHash", (PyCFunction)recursiveTypeGroupHash, METH_VARARGS, NULL}, - {"checkForHashInstability", (PyCFunction)checkForHashInstability, METH_VARARGS, checkForHashInstability_doc}, + {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroupDeepRepr", (PyCFunction)recursiveTypeGroupDeepRepr, METH_VARARGS | METH_KEYWORDS, NULL}, + {"recursiveTypeGroupHash", (PyCFunction)recursiveTypeGroupHash, METH_VARARGS | METH_KEYWORDS, NULL}, + {"checkForHashInstability", (PyCFunction)checkForHashInstability, METH_VARARGS | METH_KEYWORDS, checkForHashInstability_doc}, {"resetCompilerVisibleObjectHashCache", (PyCFunction)resetCompilerVisibleObjectHashCache, METH_VARARGS, resetCompilerVisibleObjectHashCache_doc}, - {"typeWalkRecord", (PyCFunction)typeWalkRecord, METH_VARARGS, NULL}, - {"typesAndObjectsVisibleToCompilerFrom", (PyCFunction)typesAndObjectsVisibleToCompilerFrom, METH_VARARGS, NULL}, - {"isRecursive", (PyCFunction)isRecursive, METH_VARARGS, NULL}, + {"typeWalkRecord", (PyCFunction)typeWalkRecord, METH_VARARGS | METH_KEYWORDS, NULL}, + {"typesAndObjectsVisibleToCompilerFrom", (PyCFunction)typesAndObjectsVisibleToCompilerFrom, METH_VARARGS | METH_KEYWORDS, NULL}, + {"isRecursive", (PyCFunction)isRecursive, METH_VARARGS | METH_KEYWORDS, NULL}, {"referencedTypes", (PyCFunction)referencedTypes, METH_VARARGS, NULL}, {"wantsToDefaultConstruct", (PyCFunction)wantsToDefaultConstruct, METH_VARARGS, NULL}, {"all_alternatives_empty", (PyCFunction)all_alternatives_empty, METH_VARARGS, NULL}, diff --git a/typed_python/compiler/python_to_native_converter.py b/typed_python/compiler/python_to_native_converter.py index d73392792..796ed4ac6 100644 --- a/typed_python/compiler/python_to_native_converter.py +++ b/typed_python/compiler/python_to_native_converter.py @@ -124,12 +124,12 @@ def extract_new_function_definitions(self): return res - def identityHashToLinkerName(self, name, identityHash, prefix="tp."): + def identityHashToLinkerName(self, name, compilerHash, prefix="tp."): assert isinstance(name, str) - assert isinstance(identityHash, str) + assert isinstance(compilerHash, str) assert isinstance(prefix, str) - return prefix + name + "." + identityHash + return prefix + name + "." + compilerHash def createConversionContext( self, @@ -626,9 +626,9 @@ def hashObjectToIdentity(self, hashable, isModuleVal=False): return res if isinstance(hashable, Wrapper): - return hashable.identityHash() + return hashable.compilerHash() - return Hash(_types.identityHash(hashable)) + return Hash(_types.compilerHash(hashable)) def hashGlobals(self, funcGlobals, code, funcGlobalsFromCells): """Hash a given piece of code's accesses to funcGlobals. @@ -704,7 +704,7 @@ def convert( input_types = tuple([typedPythonTypeToTypeWrapper(i) for i in input_types]) - identityHash = ( + compilerHash = ( Hash.from_integer(1) + self.hashObjectToIdentity(( funcCode, @@ -717,9 +717,9 @@ def convert( self.hashGlobals(funcGlobals, funcCode, funcGlobalsFromCells) ) - assert not identityHash.isPoison() + assert not compilerHash.isPoison() - identity = identityHash.hexdigest + identity = compilerHash.hexdigest # determine the linkName we'll use in the NativeCompiler to identify # this particular function diff --git a/typed_python/compiler/tests/closures_test.py b/typed_python/compiler/tests/closures_test.py index 2cbb55791..65e9e4fb6 100644 --- a/typed_python/compiler/tests/closures_test.py +++ b/typed_python/compiler/tests/closures_test.py @@ -24,7 +24,7 @@ ) from typed_python.compiler.runtime import RuntimeEventVisitor, Entrypoint -from typed_python._types import refcount, identityHash +from typed_python._types import refcount, compilerHash from typed_python.test_util import currentMemUsageMb @@ -221,7 +221,7 @@ def alternatingCall(c1, c2, ct): c1 = makeAppender(aList1) c2 = makeAppender(aList2) - assert identityHash(type(c1)) == identityHash(type(c2)) + assert compilerHash(type(c1)) == compilerHash(type(c2)) self.assertEqual(type(c1), type(c2)) @@ -841,7 +841,7 @@ def f(self): C1 = makeClass(1) C2 = makeClass(2) - assert identityHash(C1.f) != identityHash(C2.f) + assert compilerHash(C1.f) != compilerHash(C2.f) @Entrypoint def callF(c): diff --git a/typed_python/compiler/tests/conversion_test.py b/typed_python/compiler/tests/conversion_test.py index f16181e17..7710b20ff 100644 --- a/typed_python/compiler/tests/conversion_test.py +++ b/typed_python/compiler/tests/conversion_test.py @@ -25,7 +25,7 @@ from typed_python import ( Function, OneOf, TupleOf, ListOf, Tuple, NamedTuple, Class, NotCompiled, Dict, _types, Compiled, Member, Final, isCompiled, ConstDict, - makeNamedTuple, UInt32, Int32, Type, identityHash, typeKnownToCompiler, checkOneOfType, + makeNamedTuple, UInt32, Int32, Type, compilerHash, typeKnownToCompiler, checkOneOfType, refcount, checkType, map ) @@ -2867,7 +2867,7 @@ def call(x): assert newCall.__code__.co_filename == call.__code__.co_filename - assert identityHash(call.__code__) == identityHash(newCall.__code__) + assert compilerHash(call.__code__) == compilerHash(newCall.__code__) def test_code_with_nested_listcomp(self): def call(x): diff --git a/typed_python/compiler/type_wrappers/wrapper.py b/typed_python/compiler/type_wrappers/wrapper.py index 863ec42ab..fb1e5f84d 100644 --- a/typed_python/compiler/type_wrappers/wrapper.py +++ b/typed_python/compiler/type_wrappers/wrapper.py @@ -110,10 +110,10 @@ def __init__(self, typeRepresentation): def isIterable(self): return "Maybe" - def identityHash(self): + def compilerHash(self): return ( - Hash(_types.identityHash(self.typeRepresentation)) - + Hash(_types.identityHash(self.interpreterTypeRepresentation)) + Hash(_types.compilerHash(self.typeRepresentation)) + + Hash(_types.compilerHash(self.interpreterTypeRepresentation)) ) def __eq__(self, other): diff --git a/typed_python/module_representation_test.py b/typed_python/module_representation_test.py index 9d9380eb7..5a226be35 100644 --- a/typed_python/module_representation_test.py +++ b/typed_python/module_representation_test.py @@ -1,5 +1,5 @@ from typed_python import ModuleRepresentation, SerializationContext -from typed_python import sha_hash, identityHash, ListOf +from typed_python import sha_hash, compilerHash, ListOf import tempfile import unittest @@ -223,7 +223,7 @@ def test_tp_functions(self): assert mr2.getDict()['f'](10) == 11 - assert identityHash(mr.getDict()['f']) != identityHash(mr2.getDict()['f']) + assert compilerHash(mr.getDict()['f']) != compilerHash(mr2.getDict()['f']) def test_tp_classes(self): with tempfile.TemporaryDirectory() as td: diff --git a/typed_python/test_util.py b/typed_python/test_util.py index 062a3a664..091b92dbd 100644 --- a/typed_python/test_util.py +++ b/typed_python/test_util.py @@ -198,7 +198,7 @@ def evaluateExprInFreshProcess(filesToWrite, expression, compilerCacheDir=None, "".join(f"import {modname};" for modname in namesToImport) + ( f"import pickle;" f"import typed_python;" - f"from typed_python._types import identityHash, recursiveTypeGroup;" + f"from typed_python._types import compilerHash, recursiveTypeGroup;" f"from typed_python import *;" f"print(repr(pickle.dumps({expression})))" ) diff --git a/typed_python/type_identity_test.py b/typed_python/type_identity_test.py index ff754759e..fa61e0d34 100644 --- a/typed_python/type_identity_test.py +++ b/typed_python/type_identity_test.py @@ -21,7 +21,7 @@ from typed_python.test_util import evaluateExprInFreshProcess, callFunctionInFreshProcess from typed_python import ( UInt64, UInt32, - ListOf, TupleOf, Tuple, NamedTuple, Dict, OneOf, Forward, identityHash, + ListOf, TupleOf, Tuple, NamedTuple, Dict, OneOf, Forward, compilerHash, Entrypoint, Class, Member, Final, TypeFunction, SerializationContext, Function, NotCompiled ) @@ -76,7 +76,7 @@ def checkHash(filesToWrite, expression): Returns: a bytes object containing the sha-hash of module.thingToGrab. """ - return Hash(evaluateExprInFreshProcess(filesToWrite, f"identityHash({expression})")) + return Hash(evaluateExprInFreshProcess(filesToWrite, f"compilerHash({expression})")) def returnSerializedValue(filesToWrite, expression, printComments=False): @@ -100,8 +100,8 @@ def test_identity_ignores_function_file_accesses(): def test_identities_of_basic_types_different(): - assert identityHash(int) != identityHash(float) - assert identityHash(TupleOf(int)) != identityHash(TupleOf(float)) + assert compilerHash(int) != compilerHash(float) + assert compilerHash(TupleOf(int)) != compilerHash(TupleOf(float)) def test_object_graph_instability_is_noticed(): @@ -139,8 +139,8 @@ def f(x: int): def g(x: int): return f(x) - identityHash(f) - identityHash(NotCompiled(f)) + compilerHash(f) + compilerHash(NotCompiled(f)) hashInstability = checkForHashInstability() @@ -160,9 +160,9 @@ def functionAccessingModuleLevelThreadLocal(): def test_identity_of_function_accessing_thread_local(): print(typesAndObjectsVisibleToCompilerFrom(type(functionAccessingModuleLevelThreadLocal))) - identityHash(functionAccessingModuleLevelThreadLocal) + compilerHash(functionAccessingModuleLevelThreadLocal) moduleLevelThreadLocal.anything = True - identityHash(functionAccessingModuleLevelThreadLocal) + compilerHash(functionAccessingModuleLevelThreadLocal) hashInstability = checkForHashInstability() @@ -172,8 +172,8 @@ def test_identity_of_function_accessing_thread_local(): def test_identity_of_method_descriptors(): - assert identityHash(ListOf(int).append) != identityHash(ListOf(float).append) - assert identityHash(ListOf(int).append) != identityHash(ListOf(int).extend) + assert compilerHash(ListOf(int).append) != compilerHash(ListOf(float).append) + assert compilerHash(ListOf(int).append) != compilerHash(ListOf(int).extend) def test_class_and_held_class_in_group(): @@ -190,34 +190,34 @@ class C(Class): def test_identity_of_register_types(): - assert isinstance(identityHash(UInt64), bytes) - assert len(identityHash(UInt64)) == 20 + assert isinstance(compilerHash(UInt64), bytes) + assert len(compilerHash(UInt64)) == 20 - assert identityHash(UInt64) != identityHash(UInt32) + assert compilerHash(UInt64) != compilerHash(UInt32) def test_identity_of_list_of(): - assert identityHash(ListOf(int)) != identityHash(ListOf(float)) - assert identityHash(ListOf(int)) == identityHash(ListOf(int)) - assert identityHash(ListOf(int)) != identityHash(TupleOf(int)) + assert compilerHash(ListOf(int)) != compilerHash(ListOf(float)) + assert compilerHash(ListOf(int)) == compilerHash(ListOf(int)) + assert compilerHash(ListOf(int)) != compilerHash(TupleOf(int)) def test_identity_of_named_tuple_and_tuple(): - assert identityHash(NamedTuple(x=int)) != identityHash(NamedTuple(x=float)) - assert identityHash(NamedTuple(x=int)) == identityHash(NamedTuple(x=int)) - assert identityHash(NamedTuple(x=int)) != identityHash(Tuple(float)) + assert compilerHash(NamedTuple(x=int)) != compilerHash(NamedTuple(x=float)) + assert compilerHash(NamedTuple(x=int)) == compilerHash(NamedTuple(x=int)) + assert compilerHash(NamedTuple(x=int)) != compilerHash(Tuple(float)) - assert identityHash(NamedTuple(x=int)) != identityHash(NamedTuple(y=int)) - assert identityHash(NamedTuple(x=int, y=float)) != identityHash(NamedTuple(y=float, x=int)) + assert compilerHash(NamedTuple(x=int)) != compilerHash(NamedTuple(y=int)) + assert compilerHash(NamedTuple(x=int, y=float)) != compilerHash(NamedTuple(y=float, x=int)) def test_identity_of_dict(): - assert identityHash(Dict(int, float)) != identityHash(Dict(int, int)) - assert identityHash(Dict(int, float)) != identityHash(Dict(float, int)) + assert compilerHash(Dict(int, float)) != compilerHash(Dict(int, int)) + assert compilerHash(Dict(int, float)) != compilerHash(Dict(float, int)) def test_identity_of_oneof(): - assert identityHash(OneOf(None, int)) != identityHash(OneOf(None, float)) + assert compilerHash(OneOf(None, int)) != compilerHash(OneOf(None, float)) def test_identity_of_recursive_types(): @@ -230,15 +230,15 @@ def test_identity_of_recursive_types(): X3 = Forward("X") X3 = X3.define(TupleOf(OneOf(float, X3))) - assert identityHash(X2) == identityHash(X) - assert identityHash(X3) != identityHash(X) + assert compilerHash(X2) == compilerHash(X) + assert compilerHash(X3) != compilerHash(X) def test_identity_of_recursive_types_2(): X = Forward("X") X = X.define(TupleOf(OneOf(int, TupleOf(X)))) - identityHash(X) + compilerHash(X) def test_identity_of_recursive_types_produced_same_way(): @@ -246,9 +246,9 @@ def make(name, T): X = Forward(name) return X.define(TupleOf(OneOf(T, X))) - assert identityHash(make("X", int)) == identityHash(make("X", int)) - assert identityHash(make("X", int)) != identityHash(make("X", float)) - assert identityHash(make("X", int)) != identityHash(make("X2", int)) + assert compilerHash(make("X", int)) == compilerHash(make("X", int)) + assert compilerHash(make("X", int)) != compilerHash(make("X", float)) + assert compilerHash(make("X", int)) != compilerHash(make("X2", int)) def test_identity_of_lambda_functions(): @@ -258,15 +258,15 @@ def makeAdder(a): # these two have the same closure type assert makeAdder(10).ClosureType == makeAdder(11).ClosureType - assert identityHash(type(makeAdder(10))) == identityHash(type(makeAdder(10))) - assert identityHash(type(makeAdder(10))) == identityHash(type(makeAdder(11))) + assert compilerHash(type(makeAdder(10))) == compilerHash(type(makeAdder(10))) + assert compilerHash(type(makeAdder(10))) == compilerHash(type(makeAdder(11))) # these two are different - assert identityHash(type(makeAdder(10))) != identityHash(type(makeAdder(10.5))) + assert compilerHash(type(makeAdder(10))) != compilerHash(type(makeAdder(10.5))) def test_checkHash_works(): - assert checkHash({"x.py": "A = TupleOf(int)\n"}, 'x.A') == Hash(identityHash(TupleOf(int))) + assert checkHash({"x.py": "A = TupleOf(int)\n"}, 'x.A') == Hash(compilerHash(TupleOf(int))) def test_mutually_recursive_group_basic(): @@ -400,12 +400,12 @@ def test_hash_of_oneof(): OneOf(None, Tuple(int)((1,))), ] - hashes = set([identityHash(t) for t in oneOfs]) + hashes = set([compilerHash(t) for t in oneOfs]) assert len(hashes) == len(oneOfs) def test_identityHash_of_none(): - assert not Hash(identityHash(type(None))).isPoison() + assert not Hash(compilerHash(type(None))).isPoison() def test_identityHash_of_a_typefunction(): @@ -417,7 +417,7 @@ def L(t): L2(int) - assert identityHash(L1) == identityHash(L2) + assert compilerHash(L1) == compilerHash(L2) def test_hash_of_TP_produced_lambdas_with_different_closure_types(): @@ -431,8 +431,8 @@ def f(): returnIt(2) returnIt(2.0) - assert identityHash(returnIt(1)) == identityHash(returnIt(2)) - assert identityHash(returnIt(1)) != identityHash(returnIt(2.0)) + assert compilerHash(returnIt(1)) == compilerHash(returnIt(2)) + assert compilerHash(returnIt(1)) != compilerHash(returnIt(2.0)) def test_hash_of_native_lambdas_with_different_closure_types(): @@ -445,8 +445,8 @@ def f(): t2 = prepareArgumentToBePassedToCompiler(Entrypoint(returnIt(2))) t3 = prepareArgumentToBePassedToCompiler(Entrypoint(returnIt(2.0))) - assert identityHash(t1) == identityHash(t2) - assert identityHash(t1) != identityHash(t3) + assert compilerHash(t1) == compilerHash(t2) + assert compilerHash(t1) != compilerHash(t3) def test_checkHash_type_functions(): @@ -519,15 +519,15 @@ def test_checkHash_references_to_typed_free_objects(): def test_hash_of_builtins(): - assert not Hash(identityHash(isinstance)).isPoison() + assert not Hash(compilerHash(isinstance)).isPoison() def test_hash_of_classObj(): class C(Class, Final): x = Member(int) - assert not Hash(identityHash(Member(int))).isPoison() - assert not Hash(identityHash(C)).isPoison() + assert not Hash(compilerHash(Member(int))).isPoison() + assert not Hash(compilerHash(C)).isPoison() MODULE = { @@ -584,7 +584,7 @@ def deserializeTwiceAndCall(rep): def deserializeAndReturnHash(rep): - return identityHash(SerializationContext().deserialize(rep)) + return compilerHash(SerializationContext().deserialize(rep)) def deserializeTwiceAndConfirmEquivalent(rep): @@ -639,8 +639,13 @@ def test_serialization_of_anonymous_functions_preserves_references(): def test_hash_stability(): idHash = evaluateExprInFreshProcess({ +<<<<<<< HEAD 'x.py': 'from typed_python.compiler.native_compiler.native_ast import NamedCallTarget\n' }, 'identityHash(x.NamedCallTarget)') +======= + 'x.py': 'from typed_python.compiler.native_ast import NamedCallTarget\n' + }, 'compilerHash(x.NamedCallTarget)') +>>>>>>> d4b04a6d... Add a 'VisibilityType' flag to MRTG. Refs #439. ser = returnSerializedValue({ 'x.py': 'from typed_python.compiler.native_compiler.native_ast import NamedCallTarget\n' }, 'x.NamedCallTarget', printComments=True) @@ -688,11 +693,11 @@ def test_identity_of_entrypointed_functions(): def f(): return 0 - assert identityHash(Function(f)) != identityHash(Entrypoint(f)) + assert compilerHash(Function(f)) != compilerHash(Entrypoint(f)) def test_identity_of_singleton_classes(): - assert identityHash(A) != identityHash(B) + assert compilerHash(A) != compilerHash(B) def test_type_walk_for_named_tuple_subclass(): @@ -703,6 +708,7 @@ def f(self): print(typeWalkRecord(N)) +<<<<<<< HEAD def test_module_hash_magic_value(): with tempfile.TemporaryDirectory() as tempDir: @@ -832,3 +838,20 @@ def test_type_function_identity_referencing_int_in_function_only(): identityHash(RefsAValueInStaticmethodOnClass(1)) != identityHash(RefsAValueInStaticmethodOnClass(2)) ) + + +def test_recursive_type_groups_separate(): + class C: + pass + + assert recursiveTypeGroup(C, "compiler", False) is None + assert recursiveTypeGroup(C, "identity", False) is None + + g1 = recursiveTypeGroup(C, "compiler", True) + assert g1 + assert recursiveTypeGroup(C, "identity", False) is None + + g2 = recursiveTypeGroup(C, "identity", True) + assert g2 + + print(g1, g2) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 79554a397..5514a735c 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -49,7 +49,7 @@ ) from typed_python._types import ( - refcount, isRecursive, identityHash, buildPyFunctionObject, + refcount, isRecursive, compilerHash, buildPyFunctionObject, setFunctionClosure, typesAreEquivalent, recursiveTypeGroupDeepRepr, recursiveTypeGroupRepr ) @@ -1491,7 +1491,7 @@ def getSelf(self) -> B: B2 = sc.deserialize(sc.serialize(B)) - assert identityHash(B) == identityHash(B2) + assert compilerHash(B) == compilerHash(B2) instance = B2() @@ -1621,7 +1621,7 @@ def aFunction(self, x): aFunction2 = sc.deserialize(sc.serialize(aFunction)) - assert identityHash(aFunction) == identityHash(aFunction2) + assert compilerHash(aFunction) == compilerHash(aFunction2) def test_serialize_subclasses(self): sc = SerializationContext() @@ -2353,7 +2353,7 @@ def f(self): Cls2 = type(callFunctionInFreshProcess(Cls, ())) - assert identityHash(Cls).hex() == identityHash(Cls2).hex() + assert compilerHash(Cls).hex() == compilerHash(Cls2).hex() @pytest.mark.skipif( "sys.version_info.minor >= 8", @@ -2365,11 +2365,11 @@ def makeFunction(): def f(x: float): return x + 1 - return (f, identityHash(f)) + return (f, compilerHash(f)) f, identityHashOfF = callFunctionInFreshProcess(makeFunction, ()) - assert identityHash(f) == identityHashOfF + assert compilerHash(f) == identityHashOfF @pytest.mark.skipif( "sys.version_info.minor >= 8", @@ -2381,11 +2381,11 @@ def makeFunction(): def f(x=None): return x + 1 - return (f, identityHash(f)) + return (f, compilerHash(f)) f, identityHashOfF = callFunctionInFreshProcess(makeFunction, ()) - assert identityHash(f) == identityHashOfF + assert compilerHash(f) == identityHashOfF @pytest.mark.skip(reason='broken') def test_deserialize_untyped_class_in_forward_retains_identity(self): @@ -2418,7 +2418,7 @@ def pad(x): for i in range(len(l)): print(pad(l[i]), " " if l[i] == r[i] else " != ", pad(r[i])) - assert identityHash(Cls) == identityHash(Cls2) + assert compilerHash(Cls) == compilerHash(Cls2) def test_deserialize_tp_class_retains_identity(self): Cls = Forward("Cls") @@ -2431,7 +2431,7 @@ def f(self) -> Cls: Cls2 = type(callFunctionInFreshProcess(Cls, ())) - assert identityHash(Cls) == identityHash(Cls2) + assert compilerHash(Cls) == compilerHash(Cls2) def test_call_method_dispatch_on_two_versions_of_same_class_with_recursion_defined_in_host(self): Base = Forward("Base") @@ -2461,7 +2461,7 @@ def callF(x): aChild2 = SerializationContext().deserialize(someBytes) - assert identityHash(aChild) == identityHash(aChild2) + assert compilerHash(aChild) == compilerHash(aChild2) if callF(aChild) == callF(aChild2): return "OK" @@ -2495,8 +2495,8 @@ def f(self, x) -> int: child1, callF1 = callFunctionInFreshProcess(deserializeAndCall, ()) child2, callF2 = callFunctionInFreshProcess(deserializeAndCall, ()) - assert identityHash(child1) == identityHash(child2) - assert identityHash(callF1) == identityHash(callF2) + assert compilerHash(child1) == compilerHash(child2) + assert compilerHash(callF1) == compilerHash(callF2) assert type(child1) is type(child2) assert type(callF1) is type(callF2) @@ -2529,8 +2529,8 @@ def callF(x: Child): Base1, callF1 = callFunctionInFreshProcess(deserializeAndCall, ()) Base2, callF2 = callFunctionInFreshProcess(deserializeAndCall, ()) - assert identityHash(Base1) == identityHash(Base2) - assert identityHash(callF1) == identityHash(callF2) + assert compilerHash(Base1) == compilerHash(Base2) + assert compilerHash(callF1) == compilerHash(callF2) assert Base1 is Base2 assert type(callF1) is type(callF2) @@ -2546,7 +2546,7 @@ def test_identity_hash_of_lambda_doesnt_change_serialization(self): ser1 = s.serialize(f) - identityHash(f) + compilerHash(f) ser2 = s.serialize(f) @@ -2567,7 +2567,7 @@ def f(self, x) -> int: def callF(x: Base): return x.f(10) - identityHash(Base) + compilerHash(Base) def deserializeAndCall(): class Child(Base, Final): @@ -2858,7 +2858,7 @@ def test_serialization_independent_of_whether_function_is_hashed(self): s1 = s.serialize(moduleLevelFunctionUsedByExactlyOneSerializationTest) - identityHash(moduleLevelFunctionUsedByExactlyOneSerializationTest) + compilerHash(moduleLevelFunctionUsedByExactlyOneSerializationTest) s2 = s.serialize(moduleLevelFunctionUsedByExactlyOneSerializationTest) From d2a5e4a44524d5ff650603ce7033f914d9778ea0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 21 Apr 2023 16:22:07 +0000 Subject: [PATCH 02/83] Identity hash is stable during module import. Refs #439. --- typed_python/CompilerVisibleObjectVisitor.hpp | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index c7f467a90..f98d0cb22 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -403,6 +403,37 @@ class CompilerVisibleObjectVisitor { return lines; } + // check if this is a module's dict object and if that module is globally named. + static std::string isPyObjectGloballyIdentifiableModuleDict(PyObject* h) { + PyEnsureGilAcquired getTheGil; + + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + if (!PyDict_Check(h)) { + return std::string(); + } + + PyObject* package = PyDict_GetItemString(h, "__package__"); + PyObject* name = PyDict_GetItemString(h, "__name__"); + + if (package && name && PyUnicode_Check(name)) { + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, name)); + + if (!moduleObject) { + PyErr_Clear(); + return std::string(); + } + + PyObjectStealer dictObj(PyObject_GenericGetDict((PyObject*)moduleObject, nullptr)); + + if ((PyObject*)dictObj == h) { + return std::string(PyUnicode_AsUTF8(name)); + } + } + + return std::string(); + } + // is this a 'globally identifiable' py object, where we can just use its name to find it, // and where we are guaranteed that it won't change between invocations of the program? static bool isPyObjectGloballyIdentifiableAndStable(PyObject* h) { @@ -713,6 +744,7 @@ class CompilerVisibleObjectVisitor { # else visitor.visitTopo(co->co_lnotab); # endif + return; } @@ -743,18 +775,34 @@ class CompilerVisibleObjectVisitor { visitor.visitHash(ShaHash(1)); if (f->func_globals && PyDict_Check(f->func_globals)) { + bool shouldVisit = true; + + if (visibility == VisibilityType::Identity) { + std::string name = isPyObjectGloballyIdentifiableModuleDict( + f->func_globals + ); + if (name.size()) { + visitor.visitHash(ShaHash(2)); + visitor.visitName(name); + shouldVisit = false; + } + } - std::vector > dotAccesses; + if (shouldVisit) { + visitor.visitHash(ShaHash(2)); - Function::Overload::visitCompilerVisibleGlobals( - [&](std::string name, PyObject* val) { - if (!isSpecialIgnorableName(name)) { - visitor.visitNamedTopo(name, val); - } - }, - (PyCodeObject*)f->func_code, - f->func_globals - ); + std::vector > dotAccesses; + + Function::Overload::visitCompilerVisibleGlobals( + [&](std::string name, PyObject* val) { + if (!isSpecialIgnorableName(name)) { + visitor.visitNamedTopo(name, val); + } + }, + (PyCodeObject*)f->func_code, + f->func_globals + ); + } } visitor.visitHash(ShaHash(0)); From 2a4725663dba186bf4016d16211ad6f7b522fbf6 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 25 May 2023 22:36:52 +0000 Subject: [PATCH 03/83] [wip] reworking how forward resolution works. --- typed_python/ForwardType.hpp | 32 ++++++- typed_python/NoneType.hpp | 2 + typed_python/OneOfType.cpp | 91 ++++++++++++++++++++ typed_python/OneOfType.hpp | 29 ++++++- typed_python/PyInstance.cpp | 1 + typed_python/RegisterTypes.hpp | 3 + typed_python/TupleOrListOfType.cpp | 68 +++++++++++++++ typed_python/TupleOrListOfType.hpp | 41 ++++++++- typed_python/Type.cpp | 99 ++++++++++++++++++++++ typed_python/Type.hpp | 103 ++++++++++++++++++++++- typed_python/__init__.py | 43 +++++----- typed_python/_types.cpp | 111 ++++++++++++++++++------- typed_python/_types.hpp | 2 +- typed_python/type_construction_test.py | 55 ++++++++++++ 14 files changed, 623 insertions(+), 57 deletions(-) create mode 100644 typed_python/type_construction_test.py diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 7e861dfdd..b643d3be6 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -36,7 +36,7 @@ class Forward : public Type { { m_name = name; m_doc = Forward_doc; - + m_is_forward_defined = true; // deliberately don't invoke 'endOfConstructorInitialization' } @@ -68,6 +68,36 @@ class Forward : public Type { return new Forward(name, indexVal); } + Type* getTargetTransitive() { + if (!mTarget) { + return nullptr; + } + + if (!mTarget->isForward()) { + return mTarget; + } + + std::set seen; + + Type* curFwd = this; + + while (curFwd->isForward()) { + if (seen.find(curFwd) != seen.end()) { + throw std::runtime_error("Forward cycle detected."); + } + + seen.insert(curFwd); + + curFwd = ((Forward*)curFwd)->getTarget(); + + if (!curFwd) { + return nullptr; + } + } + + return curFwd; + } + Type* define(Type* target) { if (!target) { throw std::runtime_error("Can't resolve a Forward to the nullptr"); diff --git a/typed_python/NoneType.hpp b/typed_python/NoneType.hpp index 4d5751f42..b0c340805 100644 --- a/typed_python/NoneType.hpp +++ b/typed_python/NoneType.hpp @@ -97,4 +97,6 @@ class NoneType : public Type { void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { stream << "None"; } + + void postInitializeConcrete() {} }; diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index db1593dfa..b38a81cd0 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -187,3 +187,94 @@ OneOfType* OneOfType::Make(const std::vector& types, OneOfType* knownType return it->second; } + +Type* OneOfType::cloneForForwardResolutionConcrete() { + // create a 'blank' oneof type + return new OneOfType(); +} + +void OneOfType::initializeFromConcrete( + Type* forwardDefinitionOfSelf, + const std::map& groupMap +) { + OneOfType* selfT = (OneOfType*)forwardDefinitionOfSelf; + + std::vector heldTypes; + std::set seenTypes; + + std::function visit = [&](Type* t) { + if (t->getTypeCategory() == catOneOf) { + for (auto subt: ((OneOfType*)t)->getTypes()) { + visit(subt); + } + } else { + auto it = groupMap.find(t); + if (it != groupMap.end()) { + visit(it->second); + } else { + if (seenTypes.find(t) == seenTypes.end()) { + heldTypes.push_back(t); + seenTypes.insert(t); + } + } + } + }; + + for (auto t: selfT->m_types) { + visit(t); + } + + m_types = heldTypes; +} + +void OneOfType::postInitializeConcrete() { + if (!m_needs_post_init) { + return; + } + + for (auto t: m_types) { + t->postInitialize(); + } + + std::map ephemeralNames; + std::string name = computeRecursiveName(ephemeralNames); + m_name = name; + + m_size = computeBytecount(); + m_name = name; + + m_is_default_constructible = false; + + for (auto typePtr: m_types) { + if (typePtr->is_default_constructible()) { + m_is_default_constructible = true; + break; + } + } + + m_needs_post_init = false; +} + +std::string OneOfType::computeRecursiveNameConcrete(std::map& ioEphemeralNames) { + if (!m_needs_post_init) { + return m_name; + } + + std::string res = "OneOf("; + + bool first = true; + + for (auto t: m_types) { + if (first) { + first = false; + } else { + res += ", "; + } + + res += t->computeRecursiveName(ioEphemeralNames); + } + + res += ")"; + + return res; +} diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 2dbc43f1d..12cd2e41a 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -25,10 +25,18 @@ PyDoc_STRVAR(OneOf_doc, ); class OneOfType : public Type { + OneOfType() noexcept : + Type(TypeCategory::catOneOf), + m_needs_post_init(true) + { + m_doc = OneOf_doc; + } + public: OneOfType(const std::vector& types) noexcept : Type(TypeCategory::catOneOf), - m_types(types) + m_types(types), + m_needs_post_init(false) { if (m_types.size() > 255) { throw std::runtime_error("OneOf types are limited to 255 alternatives in this implementation"); @@ -37,6 +45,13 @@ class OneOfType : public Type { m_doc = OneOf_doc; endOfConstructorInitialization(); // finish initializing the type object. + + for (auto t: types) { + if (t->isForwardDefined()) { + m_is_forward_defined = true; + break; + } + } } template @@ -153,6 +168,18 @@ class OneOfType : public Type { static OneOfType* Make(const std::vector& types, OneOfType* knownType = nullptr); + Type* cloneForForwardResolutionConcrete(); + + void initializeFromConcrete( + Type* forwardDefinitionOfSelf, + const std::map& groupMap + ); + + void postInitializeConcrete(); + + std::string computeRecursiveNameConcrete(std::map& ioEphemeralNames); + private: std::vector m_types; + bool m_needs_post_init; }; diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 080d89648..91de4d2f6 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -365,6 +365,7 @@ PyObject* PyInstance::tp_new_type(PyTypeObject *subtype, PyObject *args, PyObjec if (catToProduce == Type::TypeCategory::catClass ) { return MakeClassType(nullptr, args); } if (catToProduce == Type::TypeCategory::catAlternative ) { return MakeAlternativeType(nullptr, args, kwds); } if (catToProduce == Type::TypeCategory::catSubclassOf ) { return MakeSubclassOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catForward ) { return MakeForwardType(nullptr, args, kwds); } PyErr_Format(PyExc_TypeError, "unknown TypeCategory %S", (PyObject*)subtype); return NULL; diff --git a/typed_python/RegisterTypes.hpp b/typed_python/RegisterTypes.hpp index a30f26e51..3a33e6327 100644 --- a/typed_python/RegisterTypes.hpp +++ b/typed_python/RegisterTypes.hpp @@ -281,6 +281,9 @@ class RegisterType : public Type { ) { copy_constructor(dest, src); } + + // no op + void postInitializeConcrete() {} }; diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 911f36724..2b091bb8d 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -156,6 +156,74 @@ bool TupleOrListOfType::cmp(instance_ptr left, instance_ptr right, int pyCompari return cmpResultToBoolForPyOrdering(pyComparisonOp,0); } + + +Type* TupleOrListOfType::cloneForForwardResolutionConcrete() { + if (m_is_tuple) { + return new TupleOfType(); + } else { + return new ListOfType(); + } +} + +void TupleOrListOfType::initializeFromConcrete( + Type* forwardDefinitionOfSelf, + const std::map& groupMap +) { + Type* eltType = ((TupleOrListOfType*)forwardDefinitionOfSelf)->m_element_type; + + if (eltType->isForwardDefined()) { + auto it = groupMap.find(eltType); + + if (it == groupMap.end()) { + throw std::runtime_error( + "Couldn't find a group definition for " + eltType->name() + ); + } + + m_element_type = it->second; + } else { + m_element_type = eltType; + } +} + +void TupleOrListOfType::postInitializeConcrete() { + if (!m_needs_post_initialize) { + return; + } + + // this is wrong because we're not taking into account exactly how we can see ourselves + std::map ephemeralNames; + std::string name = computeRecursiveName(ephemeralNames); + + m_needs_post_initialize = false; + + m_name = name; + m_stripped_name = ""; +} + +std::string TupleOrListOfType::computeRecursiveNameConcrete(std::map& ioEphemeralNames) { + if (!m_needs_post_initialize) { + return m_name; + } + + auto it = ioEphemeralNames.find(this); + + if (it != ioEphemeralNames.end()) { + return it->second; + } + + size_t count = ioEphemeralNames.size(); + + ioEphemeralNames[this] = "^" + format(count); + + if (m_is_tuple) { + return "ListOf(" + m_element_type->computeRecursiveName(ioEphemeralNames) + ")"; + } else { + return "TupleOf(" + m_element_type->computeRecursiveName(ioEphemeralNames) + ")"; + } +} + // static TupleOfType* TupleOfType::Make(Type* elt, TupleOfType* knownType) { PyEnsureGilAcquired getTheGil; diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 64e7c30d9..1987753ca 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -20,6 +20,15 @@ #include "Format.hpp" class TupleOrListOfType : public Type { +protected: + TupleOrListOfType(bool isTuple) : + Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), + m_element_type(nullptr), + m_is_tuple(isTuple), + m_needs_post_initialize(true) + { + } + public: class layout { public: @@ -36,11 +45,16 @@ class TupleOrListOfType : public Type { TupleOrListOfType(Type* type, bool isTuple) : Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), m_element_type(type), - m_is_tuple(isTuple) + m_is_tuple(isTuple), + m_needs_post_initialize(false) { m_size = sizeof(void*); m_is_default_constructible = true; + if (type->isForwardDefined()) { + m_is_forward_defined = true; + } + endOfConstructorInitialization(); // finish initializing the type object. } @@ -587,10 +601,21 @@ class TupleOrListOfType : public Type { } + Type* cloneForForwardResolutionConcrete(); + + void initializeFromConcrete( + Type* forwardDefinitionOfSelf, + const std::map& groupMap + ); + + void postInitializeConcrete(); + + std::string computeRecursiveNameConcrete(std::map& ioEphemeralNames); + protected: Type* m_element_type; - bool m_is_tuple; + bool m_needs_post_initialize; }; PyDoc_STRVAR(ListOf_doc, @@ -600,6 +625,12 @@ PyDoc_STRVAR(ListOf_doc, ); class ListOfType : public TupleOrListOfType { + friend class TupleOrListOfType; + + ListOfType() : TupleOrListOfType(false) + { + } + public: ListOfType(Type* type) : TupleOrListOfType(type, false) { @@ -659,6 +690,12 @@ PyDoc_STRVAR(TupleOf_doc, ); class TupleOfType : public TupleOrListOfType { + friend class TupleOrListOfType; + + TupleOfType() : TupleOrListOfType(true) + { + } + public: TupleOfType(Type* type) : TupleOrListOfType(type, true) { diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 77bf22c00..c475b99bf 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -408,6 +408,105 @@ void Type::forwardResolvedTo(Forward* forward, Type* resolvedTo) { } + +// try to resolve this forward type. If we can't, we'll throw an exception. On exit, +// we will have thrown, or m_forward_resolves_to will be populated. +void Type::attemptToResolve() { + if (m_forward_resolves_to) { + return; + } + + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + std::set toVisit; + toVisit.insert(this); + + // walk the graph and determine all forwards that are not resolved + while (toVisit.size()) { + Type* toCheck = *toVisit.begin(); + toVisit.erase(toCheck); + + if (toCheck->isForwardDefined() && !toCheck->m_forward_resolves_to) { + if (typesNeedingResolution.find(toCheck) == typesNeedingResolution.end()) { + typesNeedingResolution.insert(toCheck); + + toCheck->visitReferencedTypes([&](Type* subtype) { + if (subtype->isForwardDefined() && !subtype->m_forward_resolves_to) { + toVisit.insert(subtype); + } + }); + } + } + } + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget()) { + throw std::runtime_error( + "Forward defined as " + t->nameWithModule() + " has not been defined." + ); + } + } + + // for each type that we have defined in our graph, what type are we mapping it to? + // for Forwards of primitive types like ListOf or TupleOf that can see themselves, this + // this will be a 'named copy' of the type depending on the Forward name. + std::map resolutionMapping; + + // for each type we end up defining, which type did it come from and what name + // are we giving it? + std::map resolutionSource; + + // allocate new target type bodies for all regular types in the graph + for (auto t: typesNeedingResolution) { + if (!t->isForward()) { + // we're resolving to this type directly + resolutionMapping[t] = t->cloneForForwardResolution(); + resolutionSource[resolutionMapping[t]] = t; + } + } + + // now ensure that the target for any forward is the underlying type + // note that we don't need a source for any of these since nobody will end up + // actually having a forward in their graph + for (auto t: typesNeedingResolution) { + if (t->isForward()) { + Forward* f = (Forward*)t; + + Type* tgt = f->getTargetTransitive(); + + if (!tgt) { + throw std::runtime_error("somehow this forward doesn't have a target"); + } + + if (typesNeedingResolution.find(tgt) == typesNeedingResolution.end()) { + resolutionMapping[t] = tgt; + } else { + // resolve this forward to whatever its target resolves to + resolutionMapping[t] = resolutionMapping[tgt]; + } + } + } + + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->initializeFrom(typeAndSource.second, resolutionMapping); + } + + for (auto typeAndTarget: resolutionMapping) { + typeAndTarget.first->m_forward_resolves_to = typeAndTarget.second; + } + + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->postInitialize(); + } + + if (!m_forward_resolves_to) { + throw std::runtime_error("Somehow, we didn't resolve???"); + } +} + + PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { int num_args = 0; if (args) diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index a99145e6b..cacaafef6 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -917,6 +917,23 @@ class Type { return m_recursive_forward_index; } + + bool isForwardDefined() const { + return m_is_forward_defined; + } + + Type* forwardResolvesTo() { + if (!m_is_forward_defined) { + return this; + } + + if (!m_forward_resolves_to) { + attemptToResolve(); + } + + return m_forward_resolves_to; + } + protected: Type(TypeCategory in_typeCategory) : m_typeCategory(in_typeCategory), @@ -931,7 +948,9 @@ class Type { m_is_recursive_forward(false), m_recursive_forward_index(-1), mTypeGroup(nullptr), - mRecursiveTypeGroupIndex(-1) + mRecursiveTypeGroupIndex(-1), + m_is_forward_defined(false), + m_forward_resolves_to(nullptr) {} TypeCategory m_typeCategory; @@ -985,4 +1004,86 @@ class Type { MutuallyRecursiveTypeGroup* mTypeGroup; int32_t mRecursiveTypeGroupIndex; + + + + + + + + + // NEW FORWARD IMPLEMENTATION + + // is there a Forward reachable in our object graph? This is a permanent feature of + // the type object + bool m_is_forward_defined; + + // if we are a Forward, what do we resolve to? This will be null if we have not + // fully resolved ourselves and will, for valid types, become not-null pointing to + // the actual type we resolve to. + Type* m_forward_resolves_to; + + // try to resolve this forward type. If we can't, we'll throw an exception. On exit, + // we will have thrown, or m_forward_resolves_to will be populated. + void attemptToResolve(); + + // initialize ourself as a copy of 'forwardDefinitionOfSelf' where none of the types + // will have a forward definition reference. The instance must have been constructed + // using 'cloneForForwardResolution'. If the recursiveNameOverride is not None, then + // use that name for the type instead of the computed name. + void initializeFrom( + Type* forwardDefinitionOfSelf, + const std::map& groupMap + ) { + this->check([&](auto& subtype) { + return subtype.initializeFromConcrete(forwardDefinitionOfSelf, groupMap); + }); + } + + void initializeFromConcrete( + Type* forwardDefinitionOfSelf, + const std::map& groupMap + ) { + throw std::runtime_error("subclasses implement"); + } + + + // produce a copy of ourself + Type* cloneForForwardResolution() { + return this->check([&](auto& subtype) { + return subtype.cloneForForwardResolutionConcrete(); + }); + } + + Type* cloneForForwardResolutionConcrete() { + throw std::runtime_error("subclasses implement"); + } + + + void postInitializeConcrete() { + throw std::runtime_error("Type " + name() + " didn't implement postInitializeConcrete"); + } + + std::string computeRecursiveNameConcrete(std::map& ioEphemeralNames) { + auto it = ioEphemeralNames.find(this); + + if (it == ioEphemeralNames.end()) { + return m_name; + } + + return it->second; + } + +public: + // finish initializing the type assuming no forward types are reachable + void postInitialize() { + this->check([&](auto& subtype) { + subtype.postInitializeConcrete(); + }); + } + std::string computeRecursiveName(std::map& ioEphemeralNames) { + return this->check([&](auto& subtype) { + return subtype.computeRecursiveNameConcrete(ioEphemeralNames); + }); + } }; diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 3a1b0ea18..5942449ce 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -53,7 +53,9 @@ deepBytecountAndSlabs, Slab, totalBytesAllocatedOnFreeStore, ModuleRepresentation, - setGilReleaseThreadLoopSleepMicroseconds + setGilReleaseThreadLoopSleepMicroseconds, + isForwardDefined, + resolveForwardDefinedType ) import typed_python._types as _types import threading @@ -71,27 +73,28 @@ EmbeddedMessage = _types.EmbeddedMessage() PyCell = _types.PyCell() -from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa +# TODO: put this back when everything works +# from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa -from typed_python.generator import Generator # noqa +# from typed_python.generator import Generator # noqa -# this has to come at the end to break import cyclic -from typed_python.lib.map import map # noqa -from typed_python.lib.pmap import pmap # noqa -from typed_python.lib.reduce import reduce # noqa +# # this has to come at the end to break import cyclic +# from typed_python.lib.map import map # noqa +# from typed_python.lib.pmap import pmap # noqa +# from typed_python.lib.reduce import reduce # noqa -_types.initializeGlobalStatics() +# _types.initializeGlobalStatics() -# start a background thread to release the GIL for us. Instead of immediately releasing the GIL, -# we prefer to release it a short time after our C code no longer needs it, in case it -# reacquires it immediately, which is very slow. -# this is needed primarily because we often can't tell whether we're about to enter code -# that acquires and releases the GIL very frequently (say, in a tight loop), which has -# a huge performance penalty (a few thousand context switches per second!) -# instead, we have a thread that checks in the background whether any thread wants us -# to release, and if so, swap it out. This can yield a 10-50x performance improvement -# when we're acquiring and releasing frequently. -gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) -gilReleaseThreadLoop.start() +# # start a background thread to release the GIL for us. Instead of immediately releasing the GIL, +# # we prefer to release it a short time after our C code no longer needs it, in case it +# # reacquires it immediately, which is very slow. +# # this is needed primarily because we often can't tell whether we're about to enter code +# # that acquires and releases the GIL very frequently (say, in a tight loop), which has +# # a huge performance penalty (a few thousand context switches per second!) +# # instead, we have a thread that checks in the background whether any thread wants us +# # to release, and if so, swap it out. This can yield a 10-50x performance improvement +# # when we're acquiring and releasing frequently. +# gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) +# gilReleaseThreadLoop.start() -_types.setGilReleaseThreadLoopSleepMicroseconds(500) +# _types.setGilReleaseThreadLoopSleepMicroseconds(500) diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 875d79957..f8e48bea2 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2493,6 +2493,47 @@ PyObject *allForwardTypesResolved(PyObject* nullValue, PyObject* args) { return incref(t->resolved() ? Py_True : Py_False); } +PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "isForwardDefined takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + PyErr_SetString(PyExc_TypeError, "first argument to 'isForwardDefined' must be a type object"); + return NULL; + } + + return incref(t->isForwardDefined() ? Py_True : Py_False); +} + +PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "resolveForwardDefinedType takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + PyErr_SetString(PyExc_TypeError, "first argument to 'resolveForwardDefinedType' must be a type object"); + return NULL; + } + + return ::translateExceptionToPyObject([&]() { + return incref( + (PyObject*) + PyInstance::typeObj( + t->forwardResolvesTo() + ) + ); + }); +} + PyObject *isSimple(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "isSimple takes 1 positional argument"); @@ -2589,21 +2630,27 @@ PyObject *wantsToDefaultConstruct(PyObject* nullValue, PyObject* args) { return incref(HeldClass::wantsToDefaultConstruct(t) ? Py_True : Py_False); } -PyObject *bytecount(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "bytecount takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); +PyObject *bytecount(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* type; + static const char *kwlist[] = {"type", NULL}; - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", (char**)kwlist, &type)) { + throw PythonExceptionSet(); + } - if (!t) { - PyErr_SetString(PyExc_TypeError, "first argument to 'bytecount' must be a type object"); - return NULL; - } + Type* t = PyInstance::unwrapTypeArgToTypePtr(type); - return PyLong_FromLong(t->bytecount()); + if (!t) { + throw std::runtime_error("type must be a TP type instance"); + } + + if (t->isForwardDefined()) { + throw std::runtime_error("type must not be a ForwardDefined type."); + } + + return PyLong_FromLong(t->bytecount()); + }); } PyObject *typesAreEquivalent(PyObject* nullValue, PyObject* args) { @@ -2825,25 +2872,25 @@ PyObject *checkForHashInstability(PyObject* nullValue, PyObject* args, PyObject* } PyObject *typeWalkRecord(PyObject* nullValue, PyObject* args, PyObject* kwargs) { - PyObject* instance; - const char* visibility = "compiler"; - static const char *kwlist[] = {"instance", "visibility", NULL}; + return translateExceptionToPyObject([&]() { + PyObject* instance; + const char* visibility = "compiler"; + static const char *kwlist[] = {"instance", "visibility", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { - throw PythonExceptionSet(); - } + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s", (char**)kwlist, &instance, &visibility)) { + throw PythonExceptionSet(); + } - VisibilityType vis; - if (std::string(visibility) == "compiler") { - vis = VisibilityType::Compiler; - } - else if (std::string(visibility) == "identity") { - vis = VisibilityType::Identity; - } else { - throw std::runtime_error("visibility must be 'compiler' or 'identity'"); - } + VisibilityType vis; + if (std::string(visibility) == "compiler") { + vis = VisibilityType::Compiler; + } + else if (std::string(visibility) == "identity") { + vis = VisibilityType::Identity; + } else { + throw std::runtime_error("visibility must be 'compiler' or 'identity'"); + } - return translateExceptionToPyObject([&]() { TypeOrPyobj obj(instance); return PyUnicode_FromString( @@ -3077,7 +3124,7 @@ PyObject *isBinaryCompatible(PyObject* nullValue, PyObject* args) { return incref(t1->isBinaryCompatibleWith(t2) ? Py_True : Py_False); } -PyObject *MakeForward(PyObject* nullValue, PyObject* args) { +PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs) { int num_args = PyTuple_Size(args); PyThreadState * ts = PyThreadState_Get(); @@ -3445,9 +3492,10 @@ static PyMethodDef module_methods[] = { {"buildPyFunctionObject", (PyCFunction)buildPyFunctionObject, METH_VARARGS | METH_KEYWORDS, NULL}, {"isPOD", (PyCFunction)isPOD, METH_VARARGS, NULL}, {"isSimple", (PyCFunction)isSimple, METH_VARARGS, NULL}, - {"bytecount", (PyCFunction)bytecount, METH_VARARGS, NULL}, + {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, + {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, + {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, {"isBinaryCompatible", (PyCFunction)isBinaryCompatible, METH_VARARGS, NULL}, - {"Forward", (PyCFunction)MakeForward, METH_VARARGS, NULL}, {"allForwardTypesResolved", (PyCFunction)allForwardTypesResolved, METH_VARARGS, NULL}, {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -3566,6 +3614,7 @@ PyInit__types(void) PyModule_AddObject(module, "PythonObjectOfType", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPythonObjectOfType))); PyModule_AddObject(module, "PythonSubclass", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPythonSubclass))); PyModule_AddObject(module, "SubclassOf", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catSubclassOf))); + PyModule_AddObject(module, "Forward", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catForward))); if (module == NULL) return NULL; diff --git a/typed_python/_types.hpp b/typed_python/_types.hpp index ed088c35b..209e4f420 100644 --- a/typed_python/_types.hpp +++ b/typed_python/_types.hpp @@ -52,7 +52,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args); PyObject *MakeClassType(PyObject* nullValue, PyObject* args); PyObject *MakeSubclassOfType(PyObject* nullValue, PyObject* args); PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwargs); - +PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs); typedef struct { PyObject_HEAD diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py new file mode 100644 index 000000000..218f4c785 --- /dev/null +++ b/typed_python/type_construction_test.py @@ -0,0 +1,55 @@ +import pytest + +from typed_python import ( + ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType +) + + + + +def test_forward_definition(): + F = Forward("X") + + assert issubclass(F, Forward) + assert isForwardDefined(F) + + +def test_forward_one_of(): + F = Forward("X") + + O = OneOf(int, F) + + assert isForwardDefined(O) + + with pytest.raises(TypeError): + bytecount(O) + + # F is now defined. + F.define(float) + + assert resolveForwardDefinedType(F) is float + + O_resolved = resolveForwardDefinedType(O) + assert not isForwardDefined(O_resolved) + + assert bytecount(O_resolved) == 1 + bytecount(float) + + +def test_recursive_tuple_of_forward(): + F = Forward("X") + + O = OneOf(None, F) + + F.define(ListOf(O)) + + F_resolved = resolveForwardDefinedType(F) + O_resolved = resolveForwardDefinedType(O) + + assert issubclass(F_resolved, ListOf) + assert not isForwardDefined(F_resolved) + assert not isForwardDefined(O_resolved) + assert F_resolved.ElementType is O_resolved + + print(O_resolved) + print(F_resolved) + From 5435b30bd15ce82bdad018e6077e05f3e9d8430f Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 01:12:27 +0000 Subject: [PATCH 04/83] [nft] corrected type-naming model --- typed_python/OneOfType.cpp | 18 ++++--- typed_python/OneOfType.hpp | 2 +- typed_python/TupleOrListOfType.cpp | 20 ++++--- typed_python/TupleOrListOfType.hpp | 2 +- typed_python/Type.hpp | 18 +++---- typed_python/TypeStack.hpp | 72 ++++++++++++++++++++++++++ typed_python/type_construction_test.py | 12 ++++- 7 files changed, 112 insertions(+), 32 deletions(-) create mode 100644 typed_python/TypeStack.hpp diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index b38a81cd0..27e272001 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -236,12 +236,10 @@ void OneOfType::postInitializeConcrete() { t->postInitialize(); } - std::map ephemeralNames; - std::string name = computeRecursiveName(ephemeralNames); + TypeStack stack; + std::string name = computeRecursiveName(stack); m_name = name; - m_size = computeBytecount(); - m_name = name; m_is_default_constructible = false; @@ -255,11 +253,19 @@ void OneOfType::postInitializeConcrete() { m_needs_post_init = false; } -std::string OneOfType::computeRecursiveNameConcrete(std::map& ioEphemeralNames) { +std::string OneOfType::computeRecursiveNameConcrete(TypeStack& typeStack) { if (!m_needs_post_init) { return m_name; } + int index = typeStack.indexOf(this); + + if (index != -1) { + return "^" + format(index); + } + + PushTypeStack addSelf(typeStack, this); + std::string res = "OneOf("; bool first = true; @@ -271,7 +277,7 @@ std::string OneOfType::computeRecursiveNameConcrete(std::map res += ", "; } - res += t->computeRecursiveName(ioEphemeralNames); + res += t->computeRecursiveName(typeStack); } res += ")"; diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 12cd2e41a..bb8d28a1e 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -177,7 +177,7 @@ class OneOfType : public Type { void postInitializeConcrete(); - std::string computeRecursiveNameConcrete(std::map& ioEphemeralNames); + std::string computeRecursiveNameConcrete(TypeStack& stack); private: std::vector m_types; diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 2b091bb8d..498b8c55b 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -193,8 +193,8 @@ void TupleOrListOfType::postInitializeConcrete() { } // this is wrong because we're not taking into account exactly how we can see ourselves - std::map ephemeralNames; - std::string name = computeRecursiveName(ephemeralNames); + TypeStack stack; + std::string name = computeRecursiveName(stack); m_needs_post_initialize = false; @@ -202,25 +202,23 @@ void TupleOrListOfType::postInitializeConcrete() { m_stripped_name = ""; } -std::string TupleOrListOfType::computeRecursiveNameConcrete(std::map& ioEphemeralNames) { +std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { if (!m_needs_post_initialize) { return m_name; } - auto it = ioEphemeralNames.find(this); + long index = stack.indexOf(this); - if (it != ioEphemeralNames.end()) { - return it->second; + if (index != -1) { + return "^" + format(index); } - size_t count = ioEphemeralNames.size(); - - ioEphemeralNames[this] = "^" + format(count); + PushTypeStack addSelf(stack, this); if (m_is_tuple) { - return "ListOf(" + m_element_type->computeRecursiveName(ioEphemeralNames) + ")"; + return "ListOf(" + m_element_type->computeRecursiveName(stack) + ")"; } else { - return "TupleOf(" + m_element_type->computeRecursiveName(ioEphemeralNames) + ")"; + return "TupleOf(" + m_element_type->computeRecursiveName(stack) + ")"; } } diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 1987753ca..968c89ead 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -610,7 +610,7 @@ class TupleOrListOfType : public Type { void postInitializeConcrete(); - std::string computeRecursiveNameConcrete(std::map& ioEphemeralNames); + std::string computeRecursiveNameConcrete(TypeStack& typeStack); protected: Type* m_element_type; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index cacaafef6..719f6df5d 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2020 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -39,6 +39,7 @@ #include "MutuallyRecursiveTypeGroup.hpp" #include "Slab.hpp" #include "DeepcopyContext.hpp" +#include "TypeStack.hpp" class SerializationBuffer; class DeserializationBuffer; @@ -1064,14 +1065,8 @@ class Type { throw std::runtime_error("Type " + name() + " didn't implement postInitializeConcrete"); } - std::string computeRecursiveNameConcrete(std::map& ioEphemeralNames) { - auto it = ioEphemeralNames.find(this); - - if (it == ioEphemeralNames.end()) { - return m_name; - } - - return it->second; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return m_name; } public: @@ -1081,9 +1076,10 @@ class Type { subtype.postInitializeConcrete(); }); } - std::string computeRecursiveName(std::map& ioEphemeralNames) { + + std::string computeRecursiveName(TypeStack& typeStack) { return this->check([&](auto& subtype) { - return subtype.computeRecursiveNameConcrete(ioEphemeralNames); + return subtype.computeRecursiveNameConcrete(typeStack); }); } }; diff --git a/typed_python/TypeStack.hpp b/typed_python/TypeStack.hpp new file mode 100644 index 000000000..b2eb5403d --- /dev/null +++ b/typed_python/TypeStack.hpp @@ -0,0 +1,72 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include + +class Type; + + +// model a call stack of types above us in a search. Clients can ask if a type +// is above us in the stack and if so, how many levels up, using 'indexOf' +// and push new types to the stack using PushTypeStack. The stack can't have +// duplicates. + +class TypeStack { +public: + long indexOf(Type* t) { + auto it = mTypeIndices.find(t); + + if (it == mTypeIndices.end()) { + return -1; + } + + return mTypeIndices.size() - it->second - 1; + } + + void push(Type* t) { + if (mTypeIndices.find(t) != mTypeIndices.end()) { + throw std::runtime_error("Type is already in the type stack"); + } + size_t s = mTypeIndices.size(); + + mTypeIndices[t] = s; + } + + void pop(Type* t) { + mTypeIndices.erase(t); + } + +private: + std::map mTypeIndices; +}; + + +class PushTypeStack { +public: + PushTypeStack(TypeStack& stack, Type* t) : mStack(stack), mT(t) { + stack.push(mT); + } + + ~PushTypeStack() { + mStack.pop(mT); + } + +private: + TypeStack& mStack; + Type* mT; +}; diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 218f4c785..0343dbb0c 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -5,8 +5,6 @@ ) - - def test_forward_definition(): F = Forward("X") @@ -35,6 +33,16 @@ def test_forward_one_of(): assert bytecount(O_resolved) == 1 + bytecount(float) +def test_recursive_definition_of_self(): + F = Forward("X") + + F.define(ListOf(F)) + + F = resolveForwardDefinedType(F) + + assert F.__name__ == 'TupleOf(^0)' + + def test_recursive_tuple_of_forward(): F = Forward("X") From aca1ac3a9043b3dc060f11c6a0c7c5964039ffe6 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 02:44:45 +0000 Subject: [PATCH 05/83] [wip] OneOf/TupleOf/ListOf use the new identity-hash mechanism to intern themselves. We also give them proper names based on their own recursive properties. --- typed_python/OneOfType.cpp | 14 ++++- typed_python/OneOfType.hpp | 4 ++ typed_python/TupleOrListOfType.cpp | 22 ++++---- typed_python/TupleOrListOfType.hpp | 4 ++ typed_python/Type.cpp | 76 ++++++++++++++++++++++++-- typed_python/Type.hpp | 49 ++++++++++++++++- typed_python/_types.cpp | 4 +- typed_python/type_construction_test.py | 26 ++++++--- 8 files changed, 168 insertions(+), 31 deletions(-) diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index 27e272001..d7350a0b0 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -227,6 +227,17 @@ void OneOfType::initializeFromConcrete( m_types = heldTypes; } +void OneOfType::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + for (long k = 0; k < m_types.size(); k++) { + auto it = groupMap.find(m_types[k]); + if (it != groupMap.end()) { + m_types[k] = it->second; + } + } +} + void OneOfType::postInitializeConcrete() { if (!m_needs_post_init) { return; @@ -236,9 +247,6 @@ void OneOfType::postInitializeConcrete() { t->postInitialize(); } - TypeStack stack; - std::string name = computeRecursiveName(stack); - m_name = name; m_size = computeBytecount(); m_is_default_constructible = false; diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index bb8d28a1e..00eb1b08f 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -179,6 +179,10 @@ class OneOfType : public Type { std::string computeRecursiveNameConcrete(TypeStack& stack); + void updateInternalTypePointersConcrete( + const std::map& groupMap + ); + private: std::vector m_types; bool m_needs_post_init; diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 498b8c55b..372df71ee 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -187,19 +187,21 @@ void TupleOrListOfType::initializeFromConcrete( } } +void TupleOrListOfType::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + auto it = groupMap.find(m_element_type); + if (it != groupMap.end()) { + m_element_type = it->second; + } +} + + void TupleOrListOfType::postInitializeConcrete() { if (!m_needs_post_initialize) { return; } - - // this is wrong because we're not taking into account exactly how we can see ourselves - TypeStack stack; - std::string name = computeRecursiveName(stack); - m_needs_post_initialize = false; - - m_name = name; - m_stripped_name = ""; } std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { @@ -216,9 +218,9 @@ std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { PushTypeStack addSelf(stack, this); if (m_is_tuple) { - return "ListOf(" + m_element_type->computeRecursiveName(stack) + ")"; - } else { return "TupleOf(" + m_element_type->computeRecursiveName(stack) + ")"; + } else { + return "ListOf(" + m_element_type->computeRecursiveName(stack) + ")"; } } diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 968c89ead..ca7dfdd74 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -612,6 +612,10 @@ class TupleOrListOfType : public Type { std::string computeRecursiveNameConcrete(TypeStack& typeStack); + void updateInternalTypePointersConcrete( + const std::map& groupMap + ); + protected: Type* m_element_type; bool m_is_tuple; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index c475b99bf..5183a0ef0 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -419,6 +419,7 @@ void Type::attemptToResolve() { // do the entire type resolution process while holding the GIL PyEnsureGilAcquired getTheGil; + std::set existingReferencedTypes; std::set typesNeedingResolution; std::set toVisit; toVisit.insert(this); @@ -454,11 +455,11 @@ void Type::attemptToResolve() { // this will be a 'named copy' of the type depending on the Forward name. std::map resolutionMapping; - // for each type we end up defining, which type did it come from and what name - // are we giving it? + // for each type we end up defining, which type did it come from std::map resolutionSource; // allocate new target type bodies for all regular types in the graph + // that are forward declared. for (auto t: typesNeedingResolution) { if (!t->isForward()) { // we're resolving to this type directly @@ -482,6 +483,7 @@ void Type::attemptToResolve() { if (typesNeedingResolution.find(tgt) == typesNeedingResolution.end()) { resolutionMapping[t] = tgt; + existingReferencedTypes.insert(tgt); } else { // resolve this forward to whatever its target resolves to resolutionMapping[t] = resolutionMapping[tgt]; @@ -489,18 +491,81 @@ void Type::attemptToResolve() { } } + // copy all the types. At this point, they all know that they are not forwards + // but none of them is ready to be identity-hashed yet. They all will be in a state of + // where m_needs_post_init is true. for (auto typeAndSource: resolutionSource) { typeAndSource.first->initializeFrom(typeAndSource.second, resolutionMapping); } - for (auto typeAndTarget: resolutionMapping) { - typeAndTarget.first->m_forward_resolves_to = typeAndTarget.second; + // cause each type to recompute its name. We have to do this in a single pass + // without any types being aware of their final name before we run the post initialize + // step. Otherwise, it's possible for the names to depend on the order of initialization + // and we want them to be stable. + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->recomputeNamePostInitialization(); } + // cause each type to post-initialize itself, which lets it update internal bytecounts etc for (auto typeAndSource: resolutionSource) { typeAndSource.first->postInitialize(); } + // now internalize the types by their hash. For each type, we compute a hash + // and then look to see if we've seen it before. We build a lookup table from + // each existing type to the internalized type, and then do the same process we did + // above to map any roots across. + + std::map resolvedToInternal; // each 'resolved' type what should it be replaced with + + std::set newTypes; + std::set redundantTypes; + + for (auto typeAndSource: resolutionSource) { + ShaHash h = typeAndSource.first->identityHash(); + + auto it = mInternalizedIdentityHashToType.find(h); + if (it == mInternalizedIdentityHashToType.end()) { + // this is a new type. We're going to put it in the memo and + // update it so that it ponts + newTypes.insert(typeAndSource.first); + } else { + resolvedToInternal[typeAndSource.first] = it->second; + redundantTypes.insert(typeAndSource.first); + } + } + + // update all the new types to no longer look at the redundant types + for (auto t: newTypes) { + t->updateInternalTypePointers(resolvedToInternal); + mInternalizedIdentityHashToType[t->identityHash()] = t; + resolvedToInternal[t] = t; + } + + for (auto t: redundantTypes) { + t->markRedundant(); + } + + // tell each source type which type it actually resolves to. We're holding the GIL so + // nobody should see anything until we finish this process. + for (auto typeAndTarget: resolutionMapping) { + auto it = resolvedToInternal.find(typeAndTarget.second); + if (it == resolvedToInternal.end()) { + // this must be an existing type + if (existingReferencedTypes.find(typeAndTarget.second) == existingReferencedTypes.end()) { + throw std::runtime_error( + "Failed to get an internalization of Type " + + typeAndTarget.second->name() + " and it wasn't a pre-existing type either." + ); + } + + // we resolve to it directly + typeAndTarget.first->m_forward_resolves_to = typeAndTarget.second; + } else { + typeAndTarget.first->m_forward_resolves_to = it->second; + } + } + if (!m_forward_resolves_to) { throw std::runtime_error("Somehow, we didn't resolve???"); } @@ -529,3 +594,6 @@ PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { curResolver = resolver; return curResolver; } + +//static +std::map Type::mInternalizedIdentityHashToType; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 719f6df5d..af93512f2 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -612,7 +612,7 @@ class Type { return false; } - return t1->compilerHash() == t2->compilerHash(); + return t1->identityHash() == t2->identityHash(); } //this checks _strict_ subclass. X is not a subclass of itself. @@ -881,7 +881,7 @@ class Type { type that's in our group. This gives us a unique signature for the group. The final hash is then that hash plus our id within the group. ******/ - ShaHash compilerHash() { + ShaHash identityHash() { // if we've never initialized our hash if (mIdentityHash == ShaHash()) { MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup( @@ -951,7 +951,8 @@ class Type { mTypeGroup(nullptr), mRecursiveTypeGroupIndex(-1), m_is_forward_defined(false), - m_forward_resolves_to(nullptr) + m_forward_resolves_to(nullptr), + m_is_redundant(false) {} TypeCategory m_typeCategory; @@ -975,6 +976,13 @@ class Type { // 'simple' types are those that have no reference to the python interpreter bool m_is_simple; + + + + + + + bool m_resolved; // were we defined as a recursive Forward type? @@ -1024,6 +1032,9 @@ class Type { // the actual type we resolve to. Type* m_forward_resolves_to; + // have we been made 'redundant'? + bool m_is_redundant; + // try to resolve this forward type. If we can't, we'll throw an exception. On exit, // we will have thrown, or m_forward_resolves_to will be populated. void attemptToResolve(); @@ -1041,6 +1052,16 @@ class Type { }); } + // this is a fully-resolved type that we created but aren't using. + // it should just leak and never be used again. + void markRedundant() { + m_is_redundant = true; + } + + bool isRedundant() { + return m_is_redundant; + } + void initializeFromConcrete( Type* forwardDefinitionOfSelf, const std::map& groupMap @@ -1048,6 +1069,20 @@ class Type { throw std::runtime_error("subclasses implement"); } + // update internal Type pointers to point into a new group + void updateInternalTypePointers( + const std::map& groupMap + ) { + this->check([&](auto& subtype) { + return subtype.updateInternalTypePointersConcrete(groupMap); + }); + } + + void updateInternalTypePointersConcrete( + const std::map& groupMap + ) { + throw std::runtime_error("subclasses implement"); + } // produce a copy of ourself Type* cloneForForwardResolution() { @@ -1069,6 +1104,14 @@ class Type { return m_name; } + static std::map mInternalizedIdentityHashToType; + + void recomputeNamePostInitialization() { + TypeStack stack; + m_name = computeRecursiveName(stack); + m_stripped_name = ""; + } + public: // finish initializing the type assuming no forward types are reachable void postInitialize() { diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index f8e48bea2..ade420347 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -3152,11 +3152,11 @@ PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs) return incref((PyObject*)PyInstance::typeObj( ::Forward::Make(name) - )); + )); } else { return incref((PyObject*)PyInstance::typeObj( ::Forward::Make() - )); + )); } } diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 0343dbb0c..c21c49d56 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -6,14 +6,14 @@ def test_forward_definition(): - F = Forward("X") + F = Forward() assert issubclass(F, Forward) assert isForwardDefined(F) def test_forward_one_of(): - F = Forward("X") + F = Forward() O = OneOf(int, F) @@ -34,20 +34,18 @@ def test_forward_one_of(): def test_recursive_definition_of_self(): - F = Forward("X") + F = Forward() F.define(ListOf(F)) F = resolveForwardDefinedType(F) - assert F.__name__ == 'TupleOf(^0)' + assert F.__name__ == 'ListOf(^0)' def test_recursive_tuple_of_forward(): - F = Forward("X") - + F = Forward() O = OneOf(None, F) - F.define(ListOf(O)) F_resolved = resolveForwardDefinedType(F) @@ -58,6 +56,16 @@ def test_recursive_tuple_of_forward(): assert not isForwardDefined(O_resolved) assert F_resolved.ElementType is O_resolved - print(O_resolved) - print(F_resolved) + assert F_resolved.__name__ == 'ListOf(OneOf(None, ^1))' + assert O_resolved.__name__ == 'OneOf(None, ListOf(^1))' + + +def test_recursive_tuple_of_forward_memoizes(): + def makeTup(): + F = Forward() + O = OneOf(None, F) + F.define(ListOf(O)) + + return resolveForwardDefinedType(F) + assert makeTup() is makeTup() From a3cceb3692a1e6c29e762cf8a05afdc25b8418f5 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 03:42:12 +0000 Subject: [PATCH 06/83] [wip] start to formalize the new model a little more. --- typed_python/CompositeType.cpp | 37 +++ typed_python/CompositeType.hpp | 33 +-- typed_python/ForwardType.hpp | 121 +------- typed_python/OneOfType.cpp | 109 ++----- typed_python/OneOfType.hpp | 38 +-- typed_python/PyForwardInstance.hpp | 23 -- typed_python/RegisterTypes.hpp | 6 +- typed_python/TupleOrListOfType.cpp | 77 ++--- typed_python/TupleOrListOfType.hpp | 36 +-- typed_python/Type.cpp | 121 -------- typed_python/Type.hpp | 153 +++------- typed_python/__init__.py | 28 +- typed_python/_types.cpp | 391 +++++++++++++------------ typed_python/type_construction_test.py | 5 + 14 files changed, 373 insertions(+), 805 deletions(-) diff --git a/typed_python/CompositeType.cpp b/typed_python/CompositeType.cpp index 0c16b901e..88072ae35 100644 --- a/typed_python/CompositeType.cpp +++ b/typed_python/CompositeType.cpp @@ -36,6 +36,7 @@ bool CompositeType::isBinaryCompatibleWithConcrete(Type* other) { return true; } +/* bool CompositeType::_updateAfterForwardTypesChanged() { bool is_default_constructible = true; size_t size = 0; @@ -70,6 +71,7 @@ bool CompositeType::_updateAfterForwardTypesChanged() { return anyChanged; } +*/ bool CompositeType::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (pyComparisonOp == Py_NE) { @@ -159,6 +161,40 @@ void CompositeType::assign(instance_ptr self, instance_ptr other) { } } +std::string NamedTuple::computeRecursiveNameConcrete(TypeStack& typeStack) { + std::string name = "NamedTuple("; + + if (m_types.size() != m_names.size()) { + throw std::logic_error("Names mismatched with types!"); + } + + for (long k = 0; k < m_types.size();k++) { + if (k) { + name += ", "; + } + name += m_names[k] + "=" + m_types[k]->computeRecursiveName(typeStack); + } + name += ")"; + + return name; +} + +std::string Tuple::computeRecursiveNameConcrete(TypeStack& typeStack) { + std::string name = "Tuple("; + + for (long k = 0; k < m_types.size();k++) { + if (k) { + name += ", "; + } + name += m_types[k]->computeRecursiveName(typeStack); + } + name += ")"; + + return name; +} + + +/* bool NamedTuple::_updateAfterForwardTypesChanged() { bool anyChanged = ((CompositeType*)this)->_updateAfterForwardTypesChanged(); @@ -209,3 +245,4 @@ bool Tuple::_updateAfterForwardTypesChanged() { return anyChanged || m_name != oldName; } +*/ \ No newline at end of file diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index ac635620e..0f0760b83 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -34,8 +34,6 @@ class CompositeType : public Type { for (long k = 0; k < names.size(); k++) { m_nameToIndex[names[k]] = k; } - - endOfConstructorInitialization(); // finish initializing the type object. } template @@ -66,8 +64,6 @@ class CompositeType : public Type { _visitContainedTypes(visitor); } - bool _updateAfterForwardTypesChanged(); - instance_ptr eltPtr(instance_ptr self, int64_t ix) const { return self + m_byte_offsets[ix]; } @@ -230,7 +226,7 @@ class CompositeType : public Type { protected: template - static subtype* MakeSubtype(const std::vector& types, const std::vector& names, subtype* knownType = nullptr) { + static subtype* MakeSubtype(const std::vector& types, const std::vector& names) { PyEnsureGilAcquired getTheGil; typedef std::pair, const std::vector > keytype; @@ -242,7 +238,7 @@ class CompositeType : public Type { it = m.insert( std::make_pair( keytype(types, names), - knownType ? knownType : new subtype(types, names) + new subtype(types, names) ) ).first; } @@ -273,22 +269,16 @@ class NamedTuple : public CompositeType { assert(types.size() == names.size()); m_doc = NamedTuple_doc; - - endOfConstructorInitialization(); // finish initializing the type object. } - bool _updateAfterForwardTypesChanged(); - - void _updateTypeMemosAfterForwardResolution() { - NamedTuple::Make(m_types, m_names, this); - } - - static NamedTuple* Make(const std::vector& types, const std::vector& names, NamedTuple* knownType = nullptr) { + static NamedTuple* Make(const std::vector& types, const std::vector& names) { if (names.size() != types.size()) { throw std::runtime_error("Names mismatched with types!"); } - return MakeSubtype(types, names, knownType); + return MakeSubtype(types, names); } + + std::string computeRecursiveNameConcrete(TypeStack& typeStack); }; PyDoc_STRVAR(Tuple_doc, @@ -304,16 +294,11 @@ class Tuple : public CompositeType { CompositeType(TypeCategory::catTuple, types, names) { m_doc = Tuple_doc; - endOfConstructorInitialization(); // finish initializing the type object. } - bool _updateAfterForwardTypesChanged(); - - void _updateTypeMemosAfterForwardResolution() { - Tuple::Make(m_types, this); + static Tuple* Make(const std::vector& types) { + return MakeSubtype(types, std::vector()); } - static Tuple* Make(const std::vector& types, Tuple* knownType=nullptr) { - return MakeSubtype(types, std::vector(), knownType); - } + std::string computeRecursiveNameConcrete(TypeStack& typeStack); }; diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index b643d3be6..cd63ec93f 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -29,15 +29,13 @@ PyDoc_STRVAR(Forward_doc, // any types that contain them can be used. class Forward : public Type { public: - Forward(std::string name, int index) : + Forward(std::string name) : Type(TypeCategory::catForward), - mTarget(nullptr), - mIndex(index) + mTarget(nullptr) { m_name = name; m_doc = Forward_doc; m_is_forward_defined = true; - // deliberately don't invoke 'endOfConstructorInitialization' } std::string moduleNameConcrete() { @@ -61,11 +59,7 @@ class Forward : public Type { } static Forward* Make(std::string name) { - static std::atomic index; - - int64_t indexVal = index++; - - return new Forward(name, indexVal); + return new Forward(name); } Type* getTargetTransitive() { @@ -103,94 +97,11 @@ class Forward : public Type { throw std::runtime_error("Can't resolve a Forward to the nullptr"); } - while (target->getTypeCategory() == TypeCategory::catForward) { - if (((Forward*)target)->getTarget()) { - target = ((Forward*)target)->getTarget(); - } else { - break; - } - } - - if (target == this) { - throw std::runtime_error("Can't resolve a forward to itself!"); - } - - if (target == mTarget) { - return mTarget; - } - - bool tgtIsForward = target->getTypeCategory() == TypeCategory::catForward; - - if (!tgtIsForward) { - // check the containment graph. If we are a forward that's - // directly contained by the target, then we shouldn't allow this - // definition because we'd be infinitely large. - auto& containedForwards = target->getContainedForwards(); - - if (containedForwards.find(this) != containedForwards.end()) { - throw std::runtime_error("Can't resolve forward " + m_name + " to " + - target->name() + " because it would create a type-containment cycle" + - " (specifically, the type would have an infinite bytecount because it 'contains' itself)." - ); - } - - bool thisIsRecursive = target->getReferencedForwards().find(this) != target->getReferencedForwards().end(); - - if (thisIsRecursive) { - target->setNameAndIndexForRecursiveType(m_name, mIndex); - target->_updateAfterForwardTypesChanged(); - } + if (mTarget) { + throw std::runtime_error("Forward is already resolved."); } mTarget = target; - m_resolved = true; - - // forward everyone looking at us to the new target - for (auto typePtr: m_referencing_us_indirectly) { - typePtr->forwardResolvedTo(this, target); - } - - bool anyChanged = true; - - while (anyChanged) { - anyChanged = false; - - for (auto typePtr: m_referencing_us_indirectly) { - if (typePtr->_updateAfterForwardTypesChanged()) { - anyChanged = true; - } - } - } - - std::set resolvedThisPass; - - for (auto typePtr: m_referencing_us_indirectly) { - if (typePtr->getReferencedForwards().size() == 0) { - typePtr->forwardTypesAreResolved(); - resolvedThisPass.insert(typePtr); - } - } - - // we need to order the nodes in a canonical way, which we do by - // taking the lowest-indexed 'Forward' and walking forward through the - // graph. The remaining nodes will all be non-recursive, and so we can - // visit them last, and in any order we like - - std::map forwardByIndex; - for (auto t: resolvedThisPass) { - int index = t->getRecursiveForwardIndex(); - - if (index >= 0) { - if (forwardByIndex.find(index) != forwardByIndex.end()) { - throw std::runtime_error("Somehow, a forward index got used twice?"); - } - - forwardByIndex[index] = t; - } - } - - m_name = mTarget->name(); - m_referencing_us_indirectly.clear(); return target; } @@ -216,18 +127,6 @@ class Forward : public Type { v(mTarget); } - void markIndirectForwardUse(Type* user) { - if (m_resolved) { - throw std::runtime_error("already resolved forward type " + name() + " can't be used like this."); - } - - if (user->getTypeCategory() == Type::TypeCategory::catForward) { - throw std::runtime_error("Makes no sense for a forward to be used by another forward"); - } - - m_referencing_us_indirectly.insert(user); - } - void constructor(instance_ptr self) { throw std::runtime_error("Forward types should never be explicity instantiated."); } @@ -254,16 +153,6 @@ class Forward : public Type { throw std::runtime_error("Forward types should never be explicity instantiated."); } - const std::set getReferencing() const { - return m_referencing_us_indirectly; - } private: Type* mTarget; - - // when we're trying to determine how to hash a type graph, - // it's helpful when recursive types know that they were - // defined by a forward and the index of that forward - int64_t mIndex; - - std::set m_referencing_us_indirectly; }; diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index d7350a0b0..e1e6d407f 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -36,59 +36,6 @@ bool OneOfType::isBinaryCompatibleWithConcrete(Type* other) { return true; } -bool OneOfType::_updateAfterForwardTypesChanged() { - size_t size = computeBytecount(); - std::string name = computeName(); - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool is_default_constructible = false; - - for (auto typePtr: m_types) { - if (typePtr->is_default_constructible()) { - is_default_constructible = true; - break; - } - } - - bool anyChanged = ( - size != m_size || - name != m_name || - is_default_constructible != m_is_default_constructible - ); - - m_size = size; - m_stripped_name = ""; - m_name = name; - m_stripped_name = ""; - - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - -std::string OneOfType::computeName() const { - std::string res = "OneOf("; - - bool first = true; - - for (auto t: m_types) { - if (first) { - first = false; - } else { - res += ", "; - } - - res += t->name(true); - } - - res += ")"; - - return res; -} - void OneOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { m_types[*((uint8_t*)self)]->repr(self+1, stream, isStr); } @@ -156,36 +103,34 @@ void OneOfType::assign(instance_ptr self, instance_ptr other) { } // static -OneOfType* OneOfType::Make(const std::vector& types, OneOfType* knownType) { - std::vector flat_typelist; - std::set seen; - - //make sure we only get each type once and don't have any other 'OneOfType' in there... - std::function)> visit = [&](const std::vector& subvec) { - for (auto t: subvec) { - if (t->getTypeCategory() == catOneOf) { - visit( ((OneOfType*)t)->getTypes() ); - } else if (seen.find(t) == seen.end()) { - flat_typelist.push_back(t); - seen.insert(t); - } +OneOfType* OneOfType::Make(const std::vector& types) { + bool anyForward = false; + + for (auto t: types) { + if (t->isForwardDefined()) { + anyForward = true; } - }; + } - visit(types); + if (anyForward) { + return new OneOfType(types); + } PyEnsureGilAcquired getTheGil; typedef const std::vector keytype; + static std::map memo; - static std::map m; - - auto it = m.find(flat_typelist); - if (it == m.end()) { - it = m.insert(std::make_pair(flat_typelist, knownType ? knownType : new OneOfType(flat_typelist))).first; + auto it = memo.find(types); + if (it != memo.end()) { + return it->second; } - return it->second; + OneOfType* res = new OneOfType(types); + OneOfType* concrete = (OneOfType*)res->forwardResolvesTo(); + + memo[types] = concrete; + return concrete; } Type* OneOfType::cloneForForwardResolutionConcrete() { @@ -225,6 +170,10 @@ void OneOfType::initializeFromConcrete( } m_types = heldTypes; + + if (m_types.size() > 255) { + throw std::runtime_error("OneOf types are limited to 255 alternatives in this implementation"); + } } void OneOfType::updateInternalTypePointersConcrete( @@ -262,18 +211,6 @@ void OneOfType::postInitializeConcrete() { } std::string OneOfType::computeRecursiveNameConcrete(TypeStack& typeStack) { - if (!m_needs_post_init) { - return m_name; - } - - int index = typeStack.indexOf(this); - - if (index != -1) { - return "^" + format(index); - } - - PushTypeStack addSelf(typeStack, this); - std::string res = "OneOf("; bool first = true; diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 00eb1b08f..08a09afcd 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -25,35 +25,24 @@ PyDoc_STRVAR(OneOf_doc, ); class OneOfType : public Type { + // clone initialization OneOfType() noexcept : - Type(TypeCategory::catOneOf), - m_needs_post_init(true) + Type(TypeCategory::catOneOf) { + m_needs_post_init = true; m_doc = OneOf_doc; } -public: + // forward initialization OneOfType(const std::vector& types) noexcept : - Type(TypeCategory::catOneOf), - m_types(types), - m_needs_post_init(false) + Type(TypeCategory::catOneOf), + m_types(types) { - if (m_types.size() > 255) { - throw std::runtime_error("OneOf types are limited to 255 alternatives in this implementation"); - } - + m_is_forward_defined = true; m_doc = OneOf_doc; - - endOfConstructorInitialization(); // finish initializing the type object. - - for (auto t: types) { - if (t->isForwardDefined()) { - m_is_forward_defined = true; - break; - } - } } +public: template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -78,8 +67,6 @@ class OneOfType : public Type { _visitContainedTypes(visitor); } - bool _updateAfterForwardTypesChanged(); - bool isPODConcrete() { for (auto t: m_types) { if (!t->isPOD()) { @@ -90,8 +77,6 @@ class OneOfType : public Type { return true; } - std::string computeName() const; - void deepcopyConcrete( instance_ptr dest, instance_ptr src, @@ -162,11 +147,7 @@ class OneOfType : public Type { return m_types; } - void _updateTypeMemosAfterForwardResolution() { - OneOfType::Make(m_types, this); - } - - static OneOfType* Make(const std::vector& types, OneOfType* knownType = nullptr); + static OneOfType* Make(const std::vector& types); Type* cloneForForwardResolutionConcrete(); @@ -185,5 +166,4 @@ class OneOfType : public Type { private: std::vector m_types; - bool m_needs_post_init; }; diff --git a/typed_python/PyForwardInstance.hpp b/typed_python/PyForwardInstance.hpp index 657d217fa..2c029475c 100644 --- a/typed_python/PyForwardInstance.hpp +++ b/typed_python/PyForwardInstance.hpp @@ -81,33 +81,10 @@ class PyForwardInstance : public PyInstance { return incref(PyInstance::typePtrToPyTypeRepresentation(result)); } - - static PyObject* forwardGetReferencing(PyObject* o, PyObject* args) { - if (PyTuple_Size(args) != 0) { - PyErr_SetString(PyExc_TypeError, "Forward.getReferencing takes zero arguments"); - return NULL; - } - - Forward* self_type = (Forward*)PyInstance::unwrapTypeArgToTypePtr(o); - if (!self_type) { - PyErr_SetString(PyExc_TypeError, "Forward.getReferencing unexpected error"); - return NULL; - } - - PyObject* res = PyList_New(0); - - for (auto t: self_type->getReferencing()) { - PyList_Append(res, PyInstance::typePtrToPyTypeRepresentation(t)); - } - - return res; - } - static PyMethodDef* typeMethodsConcrete(Type* t) { return new PyMethodDef [4] { {"define", (PyCFunction)PyForwardInstance::forwardDefine, METH_VARARGS | METH_CLASS, NULL}, {"get", (PyCFunction)PyForwardInstance::forwardGet, METH_VARARGS | METH_CLASS, NULL}, - {"getReferencing", (PyCFunction)PyForwardInstance::forwardGetReferencing, METH_VARARGS | METH_CLASS, NULL}, {NULL, NULL} }; } diff --git a/typed_python/RegisterTypes.hpp b/typed_python/RegisterTypes.hpp index 3a33e6327..1b186d7e7 100644 --- a/typed_python/RegisterTypes.hpp +++ b/typed_python/RegisterTypes.hpp @@ -191,8 +191,8 @@ class RegisterType : public Type { { m_size = sizeof(T); m_is_default_constructible = true; - - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = false; + m_needs_post_init = false; } bool isBinaryCompatibleWithConcrete(Type* other) { @@ -207,8 +207,6 @@ class RegisterType : public Type { return true; } - bool _updateAfterForwardTypesChanged() { return false; } - template void _visitReferencedTypes(const visitor_type& v) {} diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 372df71ee..6202936d1 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -26,21 +26,6 @@ bool TupleOrListOfType::isBinaryCompatibleWithConcrete(Type* other) { return m_element_type->isBinaryCompatibleWith(otherO->m_element_type); } -bool TupleOrListOfType::_updateAfterForwardTypesChanged() { - std::string name = (m_is_tuple ? "TupleOf(" : "ListOf(") + m_element_type->name(true) + ")"; - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool anyChanged = name != m_name; - - m_name = name; - m_stripped_name = ""; - - return anyChanged; -} - void TupleOrListOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { PushReprState isNew(stream, self); @@ -198,25 +183,13 @@ void TupleOrListOfType::updateInternalTypePointersConcrete( void TupleOrListOfType::postInitializeConcrete() { - if (!m_needs_post_initialize) { + if (!m_needs_post_init) { return; } - m_needs_post_initialize = false; + m_needs_post_init = false; } std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { - if (!m_needs_post_initialize) { - return m_name; - } - - long index = stack.indexOf(this); - - if (index != -1) { - return "^" + format(index); - } - - PushTypeStack addSelf(stack, this); - if (m_is_tuple) { return "TupleOf(" + m_element_type->computeRecursiveName(stack) + ")"; } else { @@ -225,41 +198,47 @@ std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { } // static -TupleOfType* TupleOfType::Make(Type* elt, TupleOfType* knownType) { +TupleOfType* TupleOfType::Make(Type* elt) { PyEnsureGilAcquired getTheGil; - static std::map m; - - auto it = m.find(elt); + if (elt->isForwardDefined()) { + return new TupleOfType(elt); + } - if (it == m.end()) { - if (knownType == nullptr) { - knownType = new TupleOfType(elt); - } + static std::map memo; - it = m.insert(std::make_pair(elt, knownType)).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + TupleOfType* res = new TupleOfType(elt); + TupleOfType* concrete = (TupleOfType*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } // static -ListOfType* ListOfType::Make(Type* elt, ListOfType* knownType) { +ListOfType* ListOfType::Make(Type* elt) { PyEnsureGilAcquired getTheGil; - static std::map m; + if (elt->isForwardDefined()) { + return new ListOfType(elt); + } - auto it = m.find(elt); + static std::map memo; - if (it == m.end()) { - if (knownType == nullptr) { - knownType = new ListOfType(elt); - } - - it = m.insert(std::make_pair(elt, knownType)).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + ListOfType* res = new ListOfType(elt); + ListOfType* concrete = (ListOfType*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } int64_t TupleOrListOfType::count(instance_ptr self) const { diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index ca7dfdd74..41265d957 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -21,12 +21,13 @@ class TupleOrListOfType : public Type { protected: + // this is the non-forward clone pathway TupleOrListOfType(bool isTuple) : Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), m_element_type(nullptr), - m_is_tuple(isTuple), - m_needs_post_initialize(true) + m_is_tuple(isTuple) { + m_needs_post_init = true; } public: @@ -45,17 +46,9 @@ class TupleOrListOfType : public Type { TupleOrListOfType(Type* type, bool isTuple) : Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), m_element_type(type), - m_is_tuple(isTuple), - m_needs_post_initialize(false) + m_is_tuple(isTuple) { - m_size = sizeof(void*); - m_is_default_constructible = true; - - if (type->isForwardDefined()) { - m_is_forward_defined = true; - } - - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; } bool isBinaryCompatibleWithConcrete(Type* other); @@ -76,8 +69,6 @@ class TupleOrListOfType : public Type { v.visitTopo(m_element_type); } - bool _updateAfterForwardTypesChanged(); - //serialize, but don't write a count template void serializeStream(instance_ptr self, buf_t& buffer) { @@ -619,7 +610,6 @@ class TupleOrListOfType : public Type { protected: Type* m_element_type; bool m_is_tuple; - bool m_needs_post_initialize; }; PyDoc_STRVAR(ListOf_doc, @@ -641,11 +631,7 @@ class ListOfType : public TupleOrListOfType { m_doc = ListOf_doc; } - static ListOfType* Make(Type* elt, ListOfType* knownType=nullptr); - - void _updateTypeMemosAfterForwardResolution() { - ListOfType::Make(m_element_type, this); - } + static ListOfType* Make(Type* elt); void setSizeUnsafe(instance_ptr self, size_t count); @@ -696,21 +682,17 @@ PyDoc_STRVAR(TupleOf_doc, class TupleOfType : public TupleOrListOfType { friend class TupleOrListOfType; + // clone form TupleOfType() : TupleOrListOfType(true) { } public: + // forward form TupleOfType(Type* type) : TupleOrListOfType(type, true) { m_doc = TupleOf_doc; } - void _updateTypeMemosAfterForwardResolution() { - TupleOfType::Make(m_element_type, this); - } - - // get a memoized TupleOfType. If 'knownType', then install this type - // if not already known. - static TupleOfType* Make(Type* elt, TupleOfType* knownType = nullptr); + static TupleOfType* Make(Type* elt); }; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 5183a0ef0..e6d5d7f98 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -288,127 +288,6 @@ bool Type::canConvertToTrivially(Type* otherType) { return false; } -void Type::endOfConstructorInitialization() { - visitReferencedTypes([&](Type* &t) { - while (t->getTypeCategory() == TypeCategory::catForward && ((Forward*)t)->getTarget()) { - t = ((Forward*)t)->getTarget(); - } - - if (t == this) { - return; - } - - if (t->getTypeCategory() == TypeCategory::catForward) { - m_referenced_forwards.insert((Forward*)t); - ((Forward*)t)->markIndirectForwardUse(this); - } else { - for (auto referencedT: t->getReferencedForwards()) { - m_referenced_forwards.insert(referencedT); - referencedT->markIndirectForwardUse(this); - } - } - }); - - visitContainedTypes([&](Type* t) { - if (t->getTypeCategory() == TypeCategory::catForward) { - m_contained_forwards.insert((Forward*)t); - } else { - for (auto containedT: t->getContainedForwards()) { - m_contained_forwards.insert(containedT); - } - } - }); - - if (!m_referenced_forwards.size()) { - forwardTypesAreResolved(); - } else { - updateAfterForwardTypesChanged(); - } -} - -void Type::forwardTypesAreResolved() { - m_resolved = true; - - updateAfterForwardTypesChanged(); - - if (m_is_simple) { - bool isSimple = true; - - visitReferencedTypes([&](Type* t) { if (!t->m_is_simple) isSimple = false; }); - - if (!isSimple) { - std::function markNotSimple([&](Type* t) { - if (t->m_is_simple) { - t->m_is_simple = false; - markNotSimple(t); - } - }); - - markNotSimple(this); - } - } - - if (mTypeRep) { - updateTypeRepForType(this, mTypeRep); - } -} - -void Type::forwardResolvedTo(Forward* forward, Type* resolvedTo) { - // make sure we reference this forward properly - if (m_referenced_forwards.find(forward) == m_referenced_forwards.end()) { - throw std::runtime_error( - "Internal error: we are supposed to reference forward " + - forward->name() + " but we don't have it marked." - ); - } - - // swap out the type representation. we shouldn't reference this forward any more - visitReferencedTypes([&](Type* &subtype) { - if (subtype == forward) { - subtype = resolvedTo; - } - }); - - // update the 'containment' forward graph - if (m_contained_forwards.find(forward) != m_contained_forwards.end()) { - m_contained_forwards.erase(forward); - for (auto contained: resolvedTo->getContainedForwards()) { - if (contained != forward) { - m_contained_forwards.insert(contained); - } - } - } - - m_referenced_forwards.erase(forward); - - if (resolvedTo->getTypeCategory() == TypeCategory::catForward) { - Forward* tgtForward = (Forward*)resolvedTo; - - if (tgtForward->getTarget()) { - throw std::runtime_error("Forwards must not resolve to other forwards that are already resolved!"); - } - - m_referenced_forwards.insert(tgtForward); - } else { - for (auto referenced: resolvedTo->getReferencedForwards()) { - if (referenced != forward) { - referenced->markIndirectForwardUse(this); - m_referenced_forwards.insert(referenced); - } - } - - if (m_referenced_forwards.size() == 0) { - forwardTypesAreResolved(); - } - } - - this->check([&](auto& subtype) { - subtype._updateTypeMemosAfterForwardResolution(); - }); -} - - - // try to resolve this forward type. If we can't, we'll throw an exception. On exit, // we will have thrown, or m_forward_resolves_to will be populated. void Type::attemptToResolve() { diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index af93512f2..cb7284656 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -600,19 +600,7 @@ class Type { // some pathways where the identical type object can get created // (usually through deserialization) static bool typesEquivalent(Type* t1, Type* t2) { - if (t1 == t2) { - return true; - } - - if (!t1 || !t2) { - return false; - } - - if (t1->m_typeCategory != t2->m_typeCategory) { - return false; - } - - return t1->identityHash() == t2->identityHash(); + return t1 == t2; } //this checks _strict_ subclass. X is not a subclass of itself. @@ -662,19 +650,15 @@ class Type { } void assertForwardsResolved() const { - if (!m_resolved) { - throw std::logic_error("Type " + m_name + " has unresolved forwards."); + if (m_is_forward_defined) { + throw std::logic_error("Type " + m_name + " is a forward"); } } void assertForwardsResolvedSufficientlyToInstantiate() { - this->check([&](auto& subtype) { - subtype.assertForwardsResolvedSufficientlyToInstantiateConcrete(); - }); - } - - void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { - assertForwardsResolved(); + if (m_is_forward_defined) { + throw std::logic_error("Type " + m_name + " is a forward"); + } } template @@ -739,36 +723,9 @@ class Type { }); } - // dispatch to the actual subtype - bool updateAfterForwardTypesChanged() { - return this->check([&](auto& subtype) { - return subtype._updateAfterForwardTypesChanged(); - }); - } - - // subtype-specific calculation - bool _updateAfterForwardTypesChanged() { return false; } - - // update our T::Make memos to reflect the fact that our - // types may have referred to forward that are now replaced. - // the type memos will have us listed with the Forward as one - // of the arguments to the 'Make' function, but now we'll have - // the resolved circular type dependency, and we need that memo - // to resolve to 'this' as well. - // - // subtypes are expected to specialize this function - void _updateTypeMemosAfterForwardResolution() {} - - // called when a downstream type has changed in some way. - // this may recalculate our on-disk size, our name, or some other - // feature of the type that depends on the forward delcarations - // below us. - void forwardTypesAreResolved(); - - void buildMutuallyRecursiveTypeCycle(); - // called after each type has initialized its internals - void endOfConstructorInitialization(); + // TODO: remove this + void endOfConstructorInitialization() {}; // call subtype.copy_constructor void copy_constructor(instance_ptr self, instance_ptr other); @@ -799,10 +756,6 @@ class Type { return m_is_default_constructible; } - bool resolved() const { - return m_resolved; - } - bool isBinaryCompatibleWith(Type* other); bool isBinaryCompatibleWithConcrete(Type* other) { @@ -813,22 +766,6 @@ class Type { return m_is_simple; } - const std::set& getReferencedForwards() const { - return m_referenced_forwards; - } - - const std::set& getContainedForwards() const { - return m_contained_forwards; - } - - void forwardResolvedTo(Forward* forward, Type* resolvedTo); - - void setNameAndIndexForRecursiveType(std::string nameOverride, int index) { - m_recursive_name = nameOverride; - m_is_recursive_forward = true; - m_recursive_forward_index = index; - } - // are we guaranteed we can convert to this other type at the 'Signature' level bool canConvertToTrivially(Type* otherType); @@ -910,15 +847,6 @@ class Type { return mRecursiveTypeGroupIndex; } - int64_t getRecursiveForwardIndex() { - if (!m_is_recursive_forward) { - return -1; - } - - return m_recursive_forward_index; - } - - bool isForwardDefined() const { return m_is_forward_defined; } @@ -945,16 +873,17 @@ class Type { mTypeRep(nullptr), m_base(nullptr), m_is_simple(true), - m_resolved(false), - m_is_recursive_forward(false), - m_recursive_forward_index(-1), mTypeGroup(nullptr), mRecursiveTypeGroupIndex(-1), m_is_forward_defined(false), m_forward_resolves_to(nullptr), + m_needs_post_init(false), m_is_redundant(false) {} + //todo: remove this + bool m_is_recursive_forward; + TypeCategory m_typeCategory; size_t m_size; @@ -976,34 +905,10 @@ class Type { // 'simple' types are those that have no reference to the python interpreter bool m_is_simple; - - - - - - - - bool m_resolved; - - // were we defined as a recursive Forward type? - bool m_is_recursive_forward; - - // if we are recursive, then an integer indicating the order in - // which we were resolved, which is useful when trying to - // order the type graph - int64_t m_recursive_forward_index; - enum BinaryCompatibilityCategory { Incompatible, Checking, Compatible }; std::map mIsBinaryCompatible; - // a set of forward types that we need to be resolved before we - // could be resolved - std::set m_referenced_forwards; - - // a subset of m_referenced_forwards that we directly contain - std::set m_contained_forwards; - // a sha-hash that uniquely identifies this type. If this value is // the same for two types, then they should be indistinguishable except // for pointers values. @@ -1014,15 +919,6 @@ class Type { int32_t mRecursiveTypeGroupIndex; - - - - - - - - // NEW FORWARD IMPLEMENTATION - // is there a Forward reachable in our object graph? This is a permanent feature of // the type object bool m_is_forward_defined; @@ -1035,6 +931,9 @@ class Type { // have we been made 'redundant'? bool m_is_redundant; + // do we need a post-initialization step? + bool m_needs_post_init; + // try to resolve this forward type. If we can't, we'll throw an exception. On exit, // we will have thrown, or m_forward_resolves_to will be populated. void attemptToResolve(); @@ -1097,7 +996,7 @@ class Type { void postInitializeConcrete() { - throw std::runtime_error("Type " + name() + " didn't implement postInitializeConcrete"); + throw std::runtime_error("Type " + name() + " of cat " + getTypeCategoryString() + " didn't implement postInitializeConcrete"); } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { @@ -1115,14 +1014,30 @@ class Type { public: // finish initializing the type assuming no forward types are reachable void postInitialize() { + if (!m_needs_post_init) { + return; + } + this->check([&](auto& subtype) { subtype.postInitializeConcrete(); }); } - std::string computeRecursiveName(TypeStack& typeStack) { + std::string computeRecursiveName(TypeStack& stack) { + if (!m_needs_post_init) { + return m_name; + } + + long index = stack.indexOf(this); + + if (index != -1) { + return "^" + format(index); + } + + PushTypeStack addSelf(stack, this); + return this->check([&](auto& subtype) { - return subtype.computeRecursiveNameConcrete(typeStack); + return subtype.computeRecursiveNameConcrete(stack); }); } }; diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 5942449ce..a105eb056 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -28,21 +28,21 @@ # can check this version. __untyped_serialization_version__ = 5 -from typed_python.internals import ( - Member, Final, Function, UndefinedBehaviorException, - makeNamedTuple, DisableCompiledCode, isCompiled, Held, - typeKnownToCompiler, - localVariableTypesKnownToCompiler, - checkOneOfType, - checkType -) +# from typed_python.internals import ( +# Member, Final, Function, UndefinedBehaviorException, +# makeNamedTuple, DisableCompiledCode, isCompiled, Held, +# typeKnownToCompiler, +# localVariableTypesKnownToCompiler, +# checkOneOfType, +# checkType +# ) from typed_python._types import bytecount, refcount -from typed_python.module import Module -from typed_python.type_function import TypeFunction -from typed_python.hash import sha_hash -from typed_python.SerializationContext import SerializationContext -from typed_python.type_filter import TypeFilter -from typed_python.compiler.typeof import TypeOf +# from typed_python.module import Module +# from typed_python.type_function import TypeFunction +# from typed_python.hash import sha_hash +# from typed_python.SerializationContext import SerializationContext +# from typed_python.type_filter import TypeFilter +# from typed_python.compiler.typeof import TypeOf from typed_python._types import ( Forward, TupleOf, ListOf, Tuple, NamedTuple, OneOf, ConstDict, SubclassOf, Alternative, Value, serialize, deserialize, serializeStream, deserializeStream, diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index ade420347..0a75a5726 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -41,258 +41,284 @@ #include "CompilerVisibleObjectVisitor.hpp" PyObject *MakeTupleOrListOfType(PyObject* nullValue, PyObject* args, bool isTuple) { - std::vector types; + return translateExceptionToPyObject([&]() { + std::vector types; - if (!unpackTupleToTypes(args, types)) { - return nullptr; - } + if (!unpackTupleToTypes(args, types)) { + throw PythonExceptionSet(); + } - if (types.size() != 1) { - if (isTuple) { - PyErr_SetString(PyExc_TypeError, "TupleOfType takes 1 positional argument."); - } else { - PyErr_SetString(PyExc_TypeError, "ListOfType takes 1 positional argument."); + if (types.size() != 1) { + if (isTuple) { + PyErr_SetString(PyExc_TypeError, "TupleOfType takes 1 positional argument."); + } else { + PyErr_SetString(PyExc_TypeError, "ListOfType takes 1 positional argument."); + } + throw PythonExceptionSet(); } - return NULL; - } - return incref( - (PyObject*)PyInstance::typeObj( - isTuple ? (TupleOrListOfType*)TupleOfType::Make(types[0]) : (TupleOrListOfType*)ListOfType::Make(types[0]) - ) - ); + return incref( + (PyObject*)PyInstance::typeObj( + isTuple ? (TupleOrListOfType*)TupleOfType::Make(types[0]) : (TupleOrListOfType*)ListOfType::Make(types[0]) + ) + ); + }); } PyObject *MakePointerToType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "PointerTo takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "PointerTo takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "PointerTo needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "PointerTo needs a type."); + throw PythonExceptionSet(); + } - return incref((PyObject*)PyInstance::typeObj(PointerTo::Make(t))); + return incref((PyObject*)PyInstance::typeObj(PointerTo::Make(t))); + }); } PyObject *MakeRefToType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "RefTo takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "RefTo takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "RefTo needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "RefTo needs a type."); + throw PythonExceptionSet(); + } - return translateExceptionToPyObject([&]{ - return incref((PyObject*)PyInstance::typeObj(RefTo::Make(t))); + return translateExceptionToPyObject([&]{ + return incref((PyObject*)PyInstance::typeObj(RefTo::Make(t))); + }); }); } PyObject *MakeSubclassOfType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "SubclassOf takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "SubclassOf takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "SubclassOf needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "SubclassOf needs a type."); + throw PythonExceptionSet(); + } - // types that can't be subclassed just produce values - if ((t->isClass() && !((Class*)t)->isFinal()) || - (t->isAlternative() && !((Class*)t)->isConcreteAlternative())) { - return translateExceptionToPyObject([&]{ - return incref((PyObject*)PyInstance::typeObj(SubclassOfType::Make(t))); - }); - } + // types that can't be subclassed just produce values + if ((t->isClass() && !((Class*)t)->isFinal()) || + (t->isAlternative() && !((Class*)t)->isConcreteAlternative())) { + return translateExceptionToPyObject([&]{ + return incref((PyObject*)PyInstance::typeObj(SubclassOfType::Make(t))); + }); + } - return MakeValueType(nullValue, args); + return MakeValueType(nullValue, args); + }); } PyObject *MakeTypedCellType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "TypedCell takes 1 positional argument."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "TypedCell takes 1 positional argument."); + throw PythonExceptionSet(); + } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "TypedCell needs a type."); - return NULL; - } + if (!t) { + PyErr_SetString(PyExc_TypeError, "TypedCell needs a type."); + throw PythonExceptionSet(); + } - return incref((PyObject*)PyInstance::typeObj(TypedCellType::Make(t))); + return incref((PyObject*)PyInstance::typeObj(TypedCellType::Make(t))); + }); } PyObject *MakeTupleOfType(PyObject* nullValue, PyObject* args) { - return MakeTupleOrListOfType(nullValue, args, true); + return translateExceptionToPyObject([&]() { + return MakeTupleOrListOfType(nullValue, args, true); + }); } PyObject *MakeListOfType(PyObject* nullValue, PyObject* args) { - return MakeTupleOrListOfType(nullValue, args, false); + return translateExceptionToPyObject([&]() { + return MakeTupleOrListOfType(nullValue, args, false); + }); } PyObject *MakeTupleType(PyObject* nullValue, PyObject* args) { - std::vector types; - if (!unpackTupleToTypes(args, types)) { - return NULL; - } + return translateExceptionToPyObject([&]() { + std::vector types; + if (!unpackTupleToTypes(args, types)) { + throw PythonExceptionSet(); + } - return incref((PyObject*)PyInstance::typeObj(Tuple::Make(types))); + return incref((PyObject*)PyInstance::typeObj(Tuple::Make(types))); + }); } PyObject *MakeConstDictType(PyObject* nullValue, PyObject* args) { - std::vector types; - for (long k = 0; k < PyTuple_Size(args); k++) { - PyObjectHolder item(PyTuple_GetItem(args,k)); - types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); - if (not types.back()) { - return NULL; + return translateExceptionToPyObject([&]() { + std::vector types; + for (long k = 0; k < PyTuple_Size(args); k++) { + PyObjectHolder item(PyTuple_GetItem(args,k)); + types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); + if (not types.back()) { + throw PythonExceptionSet(); + } } - } - if (types.size() != 2) { - PyErr_SetString(PyExc_TypeError, "ConstDict accepts two arguments"); - return NULL; - } + if (types.size() != 2) { + PyErr_SetString(PyExc_TypeError, "ConstDict accepts two arguments"); + throw PythonExceptionSet(); + } - PyObject* typeObj = (PyObject*)PyInstance::typeObj( - ConstDictType::Make(types[0],types[1]) - ); + PyObject* typeObj = (PyObject*)PyInstance::typeObj( + ConstDictType::Make(types[0],types[1]) + ); - return incref(typeObj); + return incref(typeObj); + }); } PyObject* MakeSetType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args)!=1) { - PyErr_SetString(PyExc_TypeError, "Set takes 1 positional arguments"); - return NULL; - } - PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); - if (!t) { - PyErr_SetString(PyExc_TypeError, "Set needs a type."); - return NULL; - } - SetType* setT = SetType::Make(t); - return incref((PyObject*)PyInstance::typeObj(setT)); + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args)!=1) { + PyErr_SetString(PyExc_TypeError, "Set takes 1 positional arguments"); + throw PythonExceptionSet(); + } + PyObjectHolder tupleItem(PyTuple_GetItem(args, 0)); + Type* t = PyInstance::unwrapTypeArgToTypePtr(tupleItem); + if (!t) { + PyErr_SetString(PyExc_TypeError, "Set needs a type."); + throw PythonExceptionSet(); + } + SetType* setT = SetType::Make(t); + return incref((PyObject*)PyInstance::typeObj(setT)); + }); } PyObject *MakeDictType(PyObject* nullValue, PyObject* args) { - std::vector types; - for (long k = 0; k < PyTuple_Size(args); k++) { - PyObjectHolder item(PyTuple_GetItem(args,k)); - types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); - if (not types.back()) { - return NULL; + return translateExceptionToPyObject([&]() { + std::vector types; + for (long k = 0; k < PyTuple_Size(args); k++) { + PyObjectHolder item(PyTuple_GetItem(args,k)); + types.push_back(PyInstance::unwrapTypeArgToTypePtr(item)); + if (not types.back()) { + throw PythonExceptionSet(); + } } - } - if (types.size() != 2) { - PyErr_SetString(PyExc_TypeError, "Dict accepts two arguments"); - return NULL; - } + if (types.size() != 2) { + PyErr_SetString(PyExc_TypeError, "Dict accepts two arguments"); + throw PythonExceptionSet(); + } - return incref( - (PyObject*)PyInstance::typeObj( - DictType::Make(types[0],types[1]) - ) - ); + return incref( + (PyObject*)PyInstance::typeObj( + DictType::Make(types[0],types[1]) + ) + ); + }); } PyObject *MakeOneOfType(PyObject* nullValue, PyObject* args) { - std::vector types; - for (long k = 0; k < PyTuple_Size(args); k++) { - PyObjectHolder item(PyTuple_GetItem(args,k)); - - Type* t = PyInstance::tryUnwrapPyInstanceToType(item); - - if (t) { - types.push_back(t); - } else { - PyErr_Format(PyExc_TypeError, - "Type arguments must be types or simple values (like ints, strings, etc.), not %S. " - "If you need a more complex value (such as a type object itself), wrap it in 'Value'.", - (PyObject*)item - ); + return translateExceptionToPyObject([&]() { + std::vector types; + for (long k = 0; k < PyTuple_Size(args); k++) { + PyObjectHolder item(PyTuple_GetItem(args,k)); + + Type* t = PyInstance::tryUnwrapPyInstanceToType(item); + + if (t) { + types.push_back(t); + } else { + PyErr_Format(PyExc_TypeError, + "Type arguments must be types or simple values (like ints, strings, etc.), not %S. " + "If you need a more complex value (such as a type object itself), wrap it in 'Value'.", + (PyObject*)item + ); - return NULL; + throw PythonExceptionSet(); + } } - } - PyObject* typeObj = (PyObject*)PyInstance::typeObj(OneOfType::Make(types)); + PyObject* typeObj = (PyObject*)PyInstance::typeObj(OneOfType::Make(types)); - return incref(typeObj); + return incref(typeObj); + }); } PyObject *MakeNamedTupleType(PyObject* nullValue, PyObject* args, PyObject* kwargs) { - if (args && PyTuple_Check(args) && PyTuple_Size(args)) { - PyErr_SetString(PyExc_TypeError, "NamedTuple takes no positional arguments."); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (args && PyTuple_Check(args) && PyTuple_Size(args)) { + PyErr_SetString(PyExc_TypeError, "NamedTuple takes no positional arguments."); + throw PythonExceptionSet(); + } - std::vector > namesAndTypes; + std::vector > namesAndTypes; - if (kwargs) { - PyObject *key, *value; - Py_ssize_t pos = 0; + if (kwargs) { + PyObject *key, *value; + Py_ssize_t pos = 0; - while (PyDict_Next(kwargs, &pos, &key, &value)) { - if (!PyUnicode_Check(key)) { - PyErr_SetString(PyExc_TypeError, "NamedTuple keywords are supposed to be strings."); - return NULL; - } + while (PyDict_Next(kwargs, &pos, &key, &value)) { + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, "NamedTuple keywords are supposed to be strings."); + throw PythonExceptionSet(); + } - namesAndTypes.push_back( - std::make_pair( - PyUnicode_AsUTF8(key), - PyInstance::unwrapTypeArgToTypePtr(value) - ) - ); + namesAndTypes.push_back( + std::make_pair( + PyUnicode_AsUTF8(key), + PyInstance::unwrapTypeArgToTypePtr(value) + ) + ); - if (not namesAndTypes.back().second) { - return NULL; + if (not namesAndTypes.back().second) { + throw PythonExceptionSet(); + } } } - } - if (PY_MINOR_VERSION <= 5) { - //we cannot rely on the ordering of 'kwargs' here because of the python version, so - //we sort it. this will be a problem for anyone running some processes using different - //python versions that share python code. - std::sort(namesAndTypes.begin(), namesAndTypes.end()); - } + if (PY_MINOR_VERSION <= 5) { + //we cannot rely on the ordering of 'kwargs' here because of the python version, so + //we sort it. this will be a problem for anyone running some processes using different + //python versions that share python code. + std::sort(namesAndTypes.begin(), namesAndTypes.end()); + } - std::vector names; - std::vector types; + std::vector names; + std::vector types; - for (auto p: namesAndTypes) { - names.push_back(p.first); - types.push_back(p.second); - } + for (auto p: namesAndTypes) { + names.push_back(p.first); + types.push_back(p.second); + } - return incref((PyObject*)PyInstance::typeObj(NamedTuple::Make(types, names))); + return incref((PyObject*)PyInstance::typeObj(NamedTuple::Make(types, names))); + }); } @@ -2473,26 +2499,6 @@ PyObject *deserializeStream(PyObject* nullValue, PyObject* args) { } } -PyObject *allForwardTypesResolved(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "allForwardTypesResolved takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); - - if (!t) { - PyErr_SetString( - PyExc_TypeError, - "first argument to 'allForwardTypesResolved' must be a type object" - ); - return NULL; - } - - return incref(t->resolved() ? Py_True : Py_False); -} - PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "isForwardDefined takes 1 positional argument"); @@ -3138,7 +3144,7 @@ PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs) } } - if (num_args > 1 || !PyUnicode_Check(PyTuple_GetItem(args,0))) { + if (num_args > 1 || (num_args == 1 && !PyUnicode_Check(PyTuple_GetItem(args,0)))) { PyErr_SetString(PyExc_TypeError, "Forward takes a zero or one string positional arguments."); return NULL; } @@ -3496,7 +3502,6 @@ static PyMethodDef module_methods[] = { {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, {"isBinaryCompatible", (PyCFunction)isBinaryCompatible, METH_VARARGS, NULL}, - {"allForwardTypesResolved", (PyCFunction)allForwardTypesResolved, METH_VARARGS, NULL}, {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupDeepRepr", (PyCFunction)recursiveTypeGroupDeepRepr, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index c21c49d56..a5c74eb0a 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -5,6 +5,11 @@ ) +def test_nonforward_definition(): + assert not isForwardDefined(OneOf(int, float)) + assert not isForwardDefined(ListOf(int)) + + def test_forward_definition(): F = Forward() From 5ae825d7f05267a05c0d9784e0f49938894eeaf0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 13:37:24 +0000 Subject: [PATCH 07/83] [wip] composite types --- typed_python/CompositeType.hpp | 107 ++++++++++++++++++++++--- typed_python/OneOfType.cpp | 28 +++---- typed_python/OneOfType.hpp | 5 +- typed_python/TupleOrListOfType.cpp | 21 +---- typed_python/TupleOrListOfType.hpp | 5 +- typed_python/Type.cpp | 3 +- typed_python/Type.hpp | 17 +--- typed_python/type_construction_test.py | 35 +++++++- 8 files changed, 148 insertions(+), 73 deletions(-) diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index 0f0760b83..da96c5205 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -22,6 +22,14 @@ class CompositeType : public Type { public: + // construct a non-forward uninitialized type + CompositeType(TypeCategory in_typeCategory) : + Type(in_typeCategory) + { + m_needs_post_init = true; + } + + // construct a forward-defined CompositeType CompositeType( TypeCategory in_typeCategory, const std::vector& types, @@ -31,6 +39,8 @@ class CompositeType : public Type { m_types(types), m_names(names) { + m_is_forward_defined = true; + for (long k = 0; k < names.size(); k++) { m_nameToIndex[names[k]] = k; } @@ -224,26 +234,79 @@ class CompositeType : public Type { return m_nameToIndex; } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_types = ((CompositeType*)forwardDefinitionOfSelf)->m_types; + m_names = ((CompositeType*)forwardDefinitionOfSelf)->m_names; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + for (long k = 0; k < m_types.size(); k++) { + auto it = groupMap.find(m_types[k]); + if (it != groupMap.end()) { + m_types[k] = it->second; + } + } + } + + void postInitializeConcrete() { + bool is_default_constructible = true; + size_t size = 0; + + m_byte_offsets.clear(); + + for (auto t: m_types) { + m_byte_offsets.push_back(size); + size += t->bytecount(); + } + + for (auto t: m_types) { + if (!t->is_default_constructible()) { + is_default_constructible = false; + } + } + + m_serialize_typecodes.clear(); + m_serialize_typecodes_to_position.clear(); + for (int i = 0; i < m_types.size(); i++) { + m_serialize_typecodes.push_back(i); + m_serialize_typecodes_to_position[i] = i; + } + + m_size = size; + m_is_default_constructible = is_default_constructible; + } + protected: template static subtype* MakeSubtype(const std::vector& types, const std::vector& names) { + bool anyForward = false; + + for (auto t: types) { + if (t->isForwardDefined()) { + anyForward = true; + } + } + + if (anyForward) { + return new subtype(types, names); + } + PyEnsureGilAcquired getTheGil; typedef std::pair, const std::vector > keytype; - static std::map m; + static std::map memo; - auto it = m.find(keytype(types, names)); - if (it == m.end()) { - it = m.insert( - std::make_pair( - keytype(types, names), - new subtype(types, names) - ) - ).first; + auto it = memo.find(keytype(types, names)); + if (it != memo.end()) { + return it->second; } - return it->second; + subtype* res = new subtype(types, names); + subtype* concrete = (subtype*)res->forwardResolvesTo(); + + memo[keytype(types, names)] = concrete; + return concrete; } std::vector m_types; @@ -259,10 +322,16 @@ PyDoc_STRVAR(NamedTuple_doc, "NamedTuple(kw)(t) -> new typed named tuple with names and types from kw, initialized from tuple t\n" "\n" "Raises TypeError if types don't match.\n" - ); +); class NamedTuple : public CompositeType { public: + // construct a non-forward NamedTuple + NamedTuple() : CompositeType(TypeCategory::catNamedTuple) + { + m_doc = NamedTuple_doc; + } + NamedTuple(const std::vector& types, const std::vector& names) : CompositeType(TypeCategory::catNamedTuple, types, names) { @@ -278,6 +347,10 @@ class NamedTuple : public CompositeType { return MakeSubtype(types, names); } + Type* cloneForForwardResolutionConcrete() { + return new NamedTuple(); + } + std::string computeRecursiveNameConcrete(TypeStack& typeStack); }; @@ -286,10 +359,16 @@ PyDoc_STRVAR(Tuple_doc, "Tuple(T1, T2, ...)(t) -> new typed tuple with types T1, T2, ..., initialized from tuple t\n" "\n" "Raises TypeError if types don't match.\n" - ); +); class Tuple : public CompositeType { public: + // construct a non-forward Tuple + Tuple() : CompositeType(TypeCategory::catTuple) + { + + } + Tuple(const std::vector& types, const std::vector& names) : CompositeType(TypeCategory::catTuple, types, names) { @@ -300,5 +379,9 @@ class Tuple : public CompositeType { return MakeSubtype(types, std::vector()); } + Type* cloneForForwardResolutionConcrete() { + return new Tuple(); + } + std::string computeRecursiveNameConcrete(TypeStack& typeStack); }; diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index e1e6d407f..fac7beb4b 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -139,12 +139,17 @@ Type* OneOfType::cloneForForwardResolutionConcrete() { } void OneOfType::initializeFromConcrete( - Type* forwardDefinitionOfSelf, - const std::map& groupMap + Type* forwardDefinitionOfSelf ) { OneOfType* selfT = (OneOfType*)forwardDefinitionOfSelf; - std::vector heldTypes; + m_types = selfT->m_types; +} + +void OneOfType::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + std::vector newTypes; std::set seenTypes; std::function visit = [&](Type* t) { @@ -158,35 +163,24 @@ void OneOfType::initializeFromConcrete( visit(it->second); } else { if (seenTypes.find(t) == seenTypes.end()) { - heldTypes.push_back(t); + newTypes.push_back(t); seenTypes.insert(t); } } } }; - for (auto t: selfT->m_types) { + for (auto t: m_types) { visit(t); } - m_types = heldTypes; + m_types = newTypes; if (m_types.size() > 255) { throw std::runtime_error("OneOf types are limited to 255 alternatives in this implementation"); } } -void OneOfType::updateInternalTypePointersConcrete( - const std::map& groupMap -) { - for (long k = 0; k < m_types.size(); k++) { - auto it = groupMap.find(m_types[k]); - if (it != groupMap.end()) { - m_types[k] = it->second; - } - } -} - void OneOfType::postInitializeConcrete() { if (!m_needs_post_init) { return; diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 08a09afcd..f64d8bfc1 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -151,10 +151,7 @@ class OneOfType : public Type { Type* cloneForForwardResolutionConcrete(); - void initializeFromConcrete( - Type* forwardDefinitionOfSelf, - const std::map& groupMap - ); + void initializeFromConcrete(Type* forwardDefinitionOfSelf); void postInitializeConcrete(); diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 6202936d1..82cc47289 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -151,25 +151,8 @@ Type* TupleOrListOfType::cloneForForwardResolutionConcrete() { } } -void TupleOrListOfType::initializeFromConcrete( - Type* forwardDefinitionOfSelf, - const std::map& groupMap -) { - Type* eltType = ((TupleOrListOfType*)forwardDefinitionOfSelf)->m_element_type; - - if (eltType->isForwardDefined()) { - auto it = groupMap.find(eltType); - - if (it == groupMap.end()) { - throw std::runtime_error( - "Couldn't find a group definition for " + eltType->name() - ); - } - - m_element_type = it->second; - } else { - m_element_type = eltType; - } +void TupleOrListOfType::initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_element_type = ((TupleOrListOfType*)forwardDefinitionOfSelf)->m_element_type; } void TupleOrListOfType::updateInternalTypePointersConcrete( diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 41265d957..fff681d63 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -594,10 +594,7 @@ class TupleOrListOfType : public Type { Type* cloneForForwardResolutionConcrete(); - void initializeFromConcrete( - Type* forwardDefinitionOfSelf, - const std::map& groupMap - ); + void initializeFromConcrete(Type* forwardDefinitionOfSelf); void postInitializeConcrete(); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index e6d5d7f98..1127861b6 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -374,7 +374,8 @@ void Type::attemptToResolve() { // but none of them is ready to be identity-hashed yet. They all will be in a state of // where m_needs_post_init is true. for (auto typeAndSource: resolutionSource) { - typeAndSource.first->initializeFrom(typeAndSource.second, resolutionMapping); + typeAndSource.first->initializeFrom(typeAndSource.second); + typeAndSource.first->updateInternalTypePointers(resolutionMapping); } // cause each type to recompute its name. We have to do this in a single pass diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index cb7284656..c0c8769f3 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -942,12 +942,9 @@ class Type { // will have a forward definition reference. The instance must have been constructed // using 'cloneForForwardResolution'. If the recursiveNameOverride is not None, then // use that name for the type instead of the computed name. - void initializeFrom( - Type* forwardDefinitionOfSelf, - const std::map& groupMap - ) { + void initializeFrom(Type* forwardDefinitionOfSelf) { this->check([&](auto& subtype) { - return subtype.initializeFromConcrete(forwardDefinitionOfSelf, groupMap); + return subtype.initializeFromConcrete(forwardDefinitionOfSelf); }); } @@ -961,17 +958,12 @@ class Type { return m_is_redundant; } - void initializeFromConcrete( - Type* forwardDefinitionOfSelf, - const std::map& groupMap - ) { + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { throw std::runtime_error("subclasses implement"); } // update internal Type pointers to point into a new group - void updateInternalTypePointers( - const std::map& groupMap - ) { + void updateInternalTypePointers(const std::map& groupMap) { this->check([&](auto& subtype) { return subtype.updateInternalTypePointersConcrete(groupMap); }); @@ -994,7 +986,6 @@ class Type { throw std::runtime_error("subclasses implement"); } - void postInitializeConcrete() { throw std::runtime_error("Type " + name() + " of cat " + getTypeCategoryString() + " didn't implement postInitializeConcrete"); } diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index a5c74eb0a..c1446a09f 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -1,13 +1,17 @@ import pytest from typed_python import ( - ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType + TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, + Tuple, NamedTuple ) def test_nonforward_definition(): assert not isForwardDefined(OneOf(int, float)) assert not isForwardDefined(ListOf(int)) + assert not isForwardDefined(TupleOf(int)) + assert not isForwardDefined(Tuple(int, int)) + assert not isForwardDefined(NamedTuple(x=int, y=float)) def test_forward_definition(): @@ -16,6 +20,12 @@ def test_forward_definition(): assert issubclass(F, Forward) assert isForwardDefined(F) + assert isForwardDefined(OneOf(int, F)) + assert isForwardDefined(ListOf(F)) + assert isForwardDefined(TupleOf(F)) + assert isForwardDefined(Tuple(int, F)) + assert isForwardDefined(NamedTuple(x=int, y=F)) + def test_forward_one_of(): F = Forward() @@ -48,7 +58,7 @@ def test_recursive_definition_of_self(): assert F.__name__ == 'ListOf(^0)' -def test_recursive_tuple_of_forward(): +def test_recursive_list_of_forward(): F = Forward() O = OneOf(None, F) F.define(ListOf(O)) @@ -65,7 +75,26 @@ def test_recursive_tuple_of_forward(): assert O_resolved.__name__ == 'OneOf(None, ListOf(^1))' -def test_recursive_tuple_of_forward_memoizes(): +def test_recursive_list_of_tuple_and_forward(): + F = Forward() + NT = NamedTuple(f=F) + F.define(ListOf(NT)) + + F_resolved = resolveForwardDefinedType(F) + NT_resolved = resolveForwardDefinedType(NT) + + assert issubclass(F_resolved, ListOf) + assert not isForwardDefined(F_resolved) + assert not isForwardDefined(NT_resolved) + assert F_resolved.ElementType is NT_resolved + assert NT_resolved.ElementTypes == (F_resolved,) + assert NT_resolved.ElementNames == ('f',) + + assert F_resolved.__name__ == 'ListOf(NamedTuple(f=^1))' + assert NT_resolved.__name__ == 'NamedTuple(f=ListOf(^1))' + + +def test_recursive_list_of_forward_memoizes(): def makeTup(): F = Forward() O = OneOf(None, F) From a5ea90f69bfa199c81323e04a5af5523873a85e0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 14:02:05 +0000 Subject: [PATCH 08/83] PointerTo, RefTo, and primitives. --- typed_python/BytesType.hpp | 4 +- typed_python/EmbeddedMessageType.hpp | 2 + typed_python/NoneType.hpp | 2 - typed_python/PointerToType.hpp | 59 ++++++++++++++++--------- typed_python/PyCellType.hpp | 15 ++----- typed_python/RefToType.hpp | 61 +++++++++++++++----------- typed_python/StringType.hpp | 4 +- typed_python/Type.hpp | 8 ++-- typed_python/ValueType.hpp | 3 ++ typed_python/type_construction_test.py | 26 ++++++++++- 10 files changed, 116 insertions(+), 68 deletions(-) diff --git a/typed_python/BytesType.hpp b/typed_python/BytesType.hpp index 374b20819..b39ff3ff5 100644 --- a/typed_python/BytesType.hpp +++ b/typed_python/BytesType.hpp @@ -36,8 +36,6 @@ class BytesType : public Type { m_name = "bytes"; m_is_default_constructible = true; m_size = sizeof(layout*); - - endOfConstructorInitialization(); // finish initializing the type object. } bool isBinaryCompatibleWithConcrete(Type* other); @@ -203,4 +201,6 @@ class BytesType : public Type { static layout* replace(layout* l, layout* old, layout* the_new, int64_t count); static layout* translate(layout* l, layout* table, layout* to_delete); static layout* maketrans(layout* from, layout* to); + + void postInitializeConcrete() {} }; diff --git a/typed_python/EmbeddedMessageType.hpp b/typed_python/EmbeddedMessageType.hpp index c8686de78..1d283f893 100644 --- a/typed_python/EmbeddedMessageType.hpp +++ b/typed_python/EmbeddedMessageType.hpp @@ -61,5 +61,7 @@ class EmbeddedMessageType : public BytesType { constructor((instance_ptr)self, outBuffer.size(), (const char*)outBuffer.buffer()); } + + void postInitializeConcrete() {} }; diff --git a/typed_python/NoneType.hpp b/typed_python/NoneType.hpp index b0c340805..7ccce6105 100644 --- a/typed_python/NoneType.hpp +++ b/typed_python/NoneType.hpp @@ -25,8 +25,6 @@ class NoneType : public Type { m_name = "None"; m_size = 0; m_is_default_constructible = true; - - endOfConstructorInitialization(); // finish initializing the type object. } bool isBinaryCompatibleWithConcrete(Type* other) { diff --git a/typed_python/PointerToType.hpp b/typed_python/PointerToType.hpp index 127845f7e..fa7bd7e81 100644 --- a/typed_python/PointerToType.hpp +++ b/typed_python/PointerToType.hpp @@ -35,12 +35,19 @@ PyDoc_STRVAR(PointerTo_doc, "The statement p[3]=v is like the C++ statement 'p[3]=v;'\n" "The expression p+3 is of type PointerTo(T), like the C++ expression 'p+3'\n" "The expression p1-p2 is of type int, like the C++ expression 'p1-p2'\n" - ); +); class PointerTo : public Type { protected: typedef void* instance; + // construct a non-forward defined pointer + PointerTo() : Type(TypeCategory::catPointerTo) + { + m_doc = PointerTo_doc; + m_needs_post_init = true; + } + public: PointerTo(Type* t) : Type(TypeCategory::catPointerTo), @@ -49,8 +56,7 @@ class PointerTo : public Type { m_size = sizeof(instance); m_is_default_constructible = true; m_doc = PointerTo_doc; - - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; } template @@ -59,42 +65,53 @@ class PointerTo : public Type { v.visitTopo(m_element_type); } - void _updateTypeMemosAfterForwardResolution() { - PointerTo::Make(m_element_type, this); - } + static PointerTo* Make(Type* elt) { + if (elt->isForwardDefined()) { + return new PointerTo(elt); + } - static PointerTo* Make(Type* elt, PointerTo* knownType = nullptr) { PyEnsureGilAcquired getTheGil; - static std::map m; + static std::map memo; - auto it = m.find(elt); - if (it == m.end()) { - it = m.insert(std::make_pair(elt, knownType ? knownType : new PointerTo(elt))).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + PointerTo* res = new PointerTo(elt); + PointerTo* concrete = (PointerTo*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } bool isPODConcrete() { return true; } - bool _updateAfterForwardTypesChanged() { - std::string name = "PointerTo(" + m_element_type->name(true) + ")"; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "PointerTo(" + m_element_type->computeRecursiveName(typeStack) + ")"; + } - if (m_is_recursive_forward) { - name = m_recursive_name; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_element_type = ((PointerTo*)forwardDefinitionOfSelf)->m_element_type; + } - bool anyChanged = name != m_name; - m_name = name; - m_stripped_name = ""; + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it = groupMap.find(m_element_type); + if (it != groupMap.end()) { + m_element_type = it->second; + } + } - return anyChanged; + Type* cloneForForwardResolutionConcrete() { + return new PointerTo(); } + void postInitializeConcrete() {} + template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/PyCellType.hpp b/typed_python/PyCellType.hpp index 5ae0c52fa..6372f3d8e 100644 --- a/typed_python/PyCellType.hpp +++ b/typed_python/PyCellType.hpp @@ -26,10 +26,9 @@ class PyCellType : public PyObjectHandleTypeBase { PyObjectHandleTypeBase(TypeCategory::catPyCell) { m_name = std::string("PyCell"); - + m_size = sizeof(layout_type*); + m_is_default_constructible = true; m_is_simple = false; - - endOfConstructorInitialization(); // finish initializing the type object. } bool isBinaryCompatibleWithConcrete(Type* other) { @@ -49,14 +48,6 @@ class PyCellType : public PyObjectHandleTypeBase { void _visitReferencedTypes(const visitor_type& visitor) { } - bool _updateAfterForwardTypesChanged() { - m_size = sizeof(layout_type*); - - m_is_default_constructible = true; - - return false; - } - typed_python_hash_type hash(instance_ptr left) { return PyObject_Hash(getPyObj(left)); } @@ -150,6 +141,8 @@ class PyCellType : public PyObjectHandleTypeBase { getPyObj(self) = PyCell_New(nullptr); } + void postInitializeConcrete() {} + static PyCellType* Make() { static PyCellType* res = new PyCellType(); diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index f808a856b..aa7b5bf81 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -22,6 +22,12 @@ class RefTo : public Type { protected: typedef void* instance; + // construct a non-forward defined refto + RefTo() : Type(TypeCategory::catRefTo) + { + m_needs_post_init = true; + } + public: RefTo(Type* t) : Type(TypeCategory::catRefTo), @@ -29,8 +35,7 @@ class RefTo : public Type { { m_size = sizeof(instance); m_is_default_constructible = false; - - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; } template @@ -39,47 +44,53 @@ class RefTo : public Type { v.visitTopo(m_element_type); } - void _updateTypeMemosAfterForwardResolution() { - RefTo::Make(m_element_type, this); - } - + static RefTo* Make(Type* elt) { + if (elt->isForwardDefined()) { + return new RefTo(elt); + } - static RefTo* Make(Type* elt, RefTo* knownType = nullptr) { PyEnsureGilAcquired getTheGil; - if (elt->getTypeCategory() != Type::TypeCategory::catHeldClass) { - throw std::runtime_error("RefTo only valid on HeldClass types"); - } - - static std::map m; + static std::map memo; - auto it = m.find(elt); - if (it == m.end()) { - it = m.insert(std::make_pair(elt, knownType ? knownType : new RefTo(elt))).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + RefTo* res = new RefTo(elt); + RefTo* concrete = (RefTo*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } bool isPODConcrete() { return true; } - bool _updateAfterForwardTypesChanged() { - std::string name = "RefTo(" + m_element_type->name(true) + ")"; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "RefTo(" + m_element_type->computeRecursiveName(typeStack) + ")"; + } - if (m_is_recursive_forward) { - name = m_recursive_name; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_element_type = ((RefTo*)forwardDefinitionOfSelf)->m_element_type; + } - bool anyChanged = name != m_name; - m_name = name; - m_stripped_name = ""; + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it = groupMap.find(m_element_type); + if (it != groupMap.end()) { + m_element_type = it->second; + } + } - return anyChanged; + Type* cloneForForwardResolutionConcrete() { + return new RefTo(); } + void postInitializeConcrete() {} + template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/StringType.hpp b/typed_python/StringType.hpp index 49f943eca..36765840a 100644 --- a/typed_python/StringType.hpp +++ b/typed_python/StringType.hpp @@ -41,8 +41,6 @@ class StringType : public Type { m_name = "str"; m_is_default_constructible = true; m_size = sizeof(void*); - - endOfConstructorInitialization(); // finish initializing the type object. } template @@ -344,4 +342,6 @@ class StringType : public Type { static bool to_int64(layout* s, int64_t* value); static bool to_float64(layout* s, double* value); + + void postInitializeConcrete() {}; }; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index c0c8769f3..ce47fa0ca 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -60,7 +60,6 @@ class Float64; class StringType; class BytesType; class OneOfType; -class SubclassOfType; class Value; class TupleOfType; class PointerTo; @@ -75,6 +74,7 @@ class ConcreteAlternative; class AlternativeMatcher; class PythonSubclass; class PythonObjectOfType; +class SubclassOfType; class Class; class HeldClass; class Function; @@ -959,7 +959,7 @@ class Type { } void initializeFromConcrete(Type* forwardDefinitionOfSelf) { - throw std::runtime_error("subclasses implement"); + throw std::runtime_error("Type " + name() + " didn't define initializeFromConcrete"); } // update internal Type pointers to point into a new group @@ -972,7 +972,7 @@ class Type { void updateInternalTypePointersConcrete( const std::map& groupMap ) { - throw std::runtime_error("subclasses implement"); + throw std::runtime_error("Type " + name() + " didn't define updateInternalTypePointersConcrete"); } // produce a copy of ourself @@ -983,7 +983,7 @@ class Type { } Type* cloneForForwardResolutionConcrete() { - throw std::runtime_error("subclasses implement"); + throw std::runtime_error("Type " + name() + " didn't define cloneForForwardResolutionConcrete"); } void postInitializeConcrete() { diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index f75f8dcea..4fa0c33f7 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -97,6 +97,7 @@ class Value : public Type { return mInstance; } + // TODO: should we allow Forward-declared Value types? static Type* Make(Instance i) { PyEnsureGilAcquired getTheGil; @@ -111,6 +112,8 @@ class Value : public Type { return it->second; } + void postInitializeConcrete() {} + private: Value(const Instance& instance); diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index c1446a09f..7989f8999 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -2,7 +2,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, - Tuple, NamedTuple + Tuple, NamedTuple, PointerTo, RefTo ) @@ -11,6 +11,8 @@ def test_nonforward_definition(): assert not isForwardDefined(ListOf(int)) assert not isForwardDefined(TupleOf(int)) assert not isForwardDefined(Tuple(int, int)) + assert not isForwardDefined(PointerTo(int)) + assert not isForwardDefined(RefTo(int)) assert not isForwardDefined(NamedTuple(x=int, y=float)) @@ -24,6 +26,8 @@ def test_forward_definition(): assert isForwardDefined(ListOf(F)) assert isForwardDefined(TupleOf(F)) assert isForwardDefined(Tuple(int, F)) + assert isForwardDefined(PointerTo(F)) + assert isForwardDefined(RefTo(F)) assert isForwardDefined(NamedTuple(x=int, y=F)) @@ -103,3 +107,23 @@ def makeTup(): return resolveForwardDefinedType(F) assert makeTup() is makeTup() + + +def test_recursive_pointer_to(): + def makeP(): + F = Forward() + F.define(PointerTo(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'PointerTo(^0)' + + +def test_recursive_ref_to(): + def makeP(): + F = Forward() + F.define(RefTo(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'RefTo(^0)' From c3aad794e90b18cf0d4b1c544b97c13d71918de1 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 15:35:41 +0000 Subject: [PATCH 09/83] Dict/Set/ConstDict --- typed_python/ConstDictType.cpp | 54 ++++++++------------------ typed_python/ConstDictType.hpp | 47 +++++++++++++++++++--- typed_python/DictType.cpp | 48 ++++++++--------------- typed_python/DictType.hpp | 53 +++++++++++++++++++++---- typed_python/SetType.cpp | 38 +++++++----------- typed_python/SetType.hpp | 49 +++++++++++++++++------ typed_python/type_construction_test.py | 26 ++++++++++++- 7 files changed, 196 insertions(+), 119 deletions(-) diff --git a/typed_python/ConstDictType.cpp b/typed_python/ConstDictType.cpp index 4f3773ca1..c7b5b750f 100644 --- a/typed_python/ConstDictType.cpp +++ b/typed_python/ConstDictType.cpp @@ -16,32 +16,6 @@ #include "AllTypes.hpp" -bool ConstDictType::_updateAfterForwardTypesChanged() { - size_t old_bytes_per_key_value_pair = m_bytes_per_key_value_pair; - - m_size = sizeof(void*); - m_is_default_constructible = true; - m_bytes_per_key = m_key->bytecount(); - m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); - m_bytes_per_key_subtree_pair = m_key->bytecount() + this->bytecount(); - - std::string name = "ConstDict(" + m_key->name(true) + "->" + m_value->name(true) + ")"; - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool anyChanged = ( - name != m_name || - m_bytes_per_key_value_pair != old_bytes_per_key_value_pair - ); - - m_name = name; - m_stripped_name = ""; - - return anyChanged; -} - bool ConstDictType::isBinaryCompatibleWithConcrete(Type* other) { if (other->getTypeCategory() != m_typeCategory) { return false; @@ -54,24 +28,28 @@ bool ConstDictType::isBinaryCompatibleWithConcrete(Type* other) { } // static -ConstDictType* ConstDictType::Make(Type* key, Type* value, ConstDictType* knownType) { +ConstDictType* ConstDictType::Make(Type* key, Type* value) { + if (key->isForwardDefined() || value->isForwardDefined()) { + return new ConstDictType(key, value); + } + PyEnsureGilAcquired getTheGil; - static std::map, ConstDictType*> m; + static std::map, ConstDictType*> memo; - auto lookup_key = std::make_pair(key,value); + auto lookup_key = std::make_pair(key, value); - auto it = m.find(lookup_key); - if (it == m.end()) { - it = m.insert( - std::make_pair( - lookup_key, - knownType ? knownType : new ConstDictType(key, value) - ) - ).first; + auto it = memo.find(lookup_key); + if (it != memo.end()) { + return it->second; } - return it->second; + ConstDictType* res = new ConstDictType(key, value); + ConstDictType* concrete = (ConstDictType*)res->forwardResolvesTo(); + + memo[lookup_key] = concrete; + + return concrete; } void ConstDictType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 0b377f440..7749b00f4 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -27,6 +27,12 @@ PyDoc_STRVAR(ConstDictType_doc, ); class ConstDictType : public Type { + ConstDictType() : Type(TypeCategory::catConstDict) + { + m_doc = ConstDictType_doc; + m_needs_post_init = true; + } + public: class layout { public: @@ -46,7 +52,7 @@ class ConstDictType : public Type { m_value(value) { m_doc = ConstDictType_doc; - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; } template @@ -66,16 +72,45 @@ class ConstDictType : public Type { v.visitTopo(m_value); } - bool _updateAfterForwardTypesChanged(); - bool isBinaryCompatibleWithConcrete(Type* other); - void _updateTypeMemosAfterForwardResolution() { - ConstDictType::Make(m_key, m_value, this); + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "ConstDict(" + + m_key->computeRecursiveName(typeStack) + + ", " + + m_value->computeRecursiveName(typeStack) + + ")"; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_key = ((ConstDictType*)forwardDefinitionOfSelf)->m_key; + m_value = ((ConstDictType*)forwardDefinitionOfSelf)->m_value; } - static ConstDictType* Make(Type* key, Type* value, ConstDictType* knownType = nullptr); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it_key = groupMap.find(m_key); + if (it_key != groupMap.end()) { + m_key = it_key->second; + } + + auto it_val = groupMap.find(m_value); + if (it_val != groupMap.end()) { + m_value = it_val->second; + } + } + + Type* cloneForForwardResolutionConcrete() { + return new ConstDictType(); + } + + void postInitializeConcrete() { + m_size = sizeof(void*); + m_is_default_constructible = true; + m_bytes_per_key = m_key->bytecount(); + m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); + } + static ConstDictType* Make(Type* key, Type* value); // hand 'visitor' each an instance_ptr for // each value. if it returns 'false', exit early. diff --git a/typed_python/DictType.cpp b/typed_python/DictType.cpp index b7591433b..307a0d5a0 100644 --- a/typed_python/DictType.cpp +++ b/typed_python/DictType.cpp @@ -16,26 +16,6 @@ #include "AllTypes.hpp" -bool DictType::_updateAfterForwardTypesChanged() { - m_size = sizeof(void*); - m_is_default_constructible = true; - m_bytes_per_key = m_key->bytecount(); - m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); - - std::string name = "Dict(" + m_key->name(true) + "->" + m_value->name(true) + ")"; - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool anyChanged = name != m_name; - - m_name = name; - m_stripped_name = ""; - - return anyChanged; -} - bool DictType::isBinaryCompatibleWithConcrete(Type* other) { if (other->getTypeCategory() != m_typeCategory) { return false; @@ -48,24 +28,28 @@ bool DictType::isBinaryCompatibleWithConcrete(Type* other) { } // static -DictType* DictType::Make(Type* key, Type* value, DictType* knownType) { +DictType* DictType::Make(Type* key, Type* value) { + if (key->isForwardDefined() || value->isForwardDefined()) { + return new DictType(key, value); + } + PyEnsureGilAcquired getTheGil; - static std::map, DictType*> m; + static std::map, DictType*> memo; - auto lookup_key = std::make_pair(key,value); + auto lookup_key = std::make_pair(key, value); - auto it = m.find(lookup_key); - if (it == m.end()) { - it = m.insert( - std::make_pair( - lookup_key, - knownType ? knownType: new DictType(key, value) - ) - ).first; + auto it = memo.find(lookup_key); + if (it != memo.end()) { + return it->second; } - return it->second; + DictType* res = new DictType(key, value); + DictType* concrete = (DictType*)res->forwardResolvesTo(); + + memo[lookup_key] = concrete; + + return concrete; } void DictType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index d6166c73b..b2be055cc 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -30,18 +30,21 @@ PyDoc_STRVAR(DictType_doc, ); class DictType : public Type { + // construct a non-forward uninitialized dict + DictType() : Type(TypeCategory::catDict) + { + m_doc = DictType_doc; + m_needs_post_init = true; + } public: + // declare a forward-defined Dict DictType(Type* key, Type* value) : Type(TypeCategory::catDict), m_key(key), m_value(value) { m_doc = DictType_doc; - endOfConstructorInitialization(); // finish initializing the type object. - } - - void _updateTypeMemosAfterForwardResolution() { - DictType::Make(m_key, m_value, this); + m_is_forward_defined = true; } template @@ -61,11 +64,45 @@ class DictType : public Type { v.visitTopo(m_value); } - bool _updateAfterForwardTypesChanged(); - bool isBinaryCompatibleWithConcrete(Type* other); - static DictType* Make(Type* key, Type* value, DictType* knownType=nullptr); + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "Dict(" + + m_key->computeRecursiveName(typeStack) + + ", " + + m_value->computeRecursiveName(typeStack) + + ")"; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_key = ((DictType*)forwardDefinitionOfSelf)->m_key; + m_value = ((DictType*)forwardDefinitionOfSelf)->m_value; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it_key = groupMap.find(m_key); + if (it_key != groupMap.end()) { + m_key = it_key->second; + } + + auto it_val = groupMap.find(m_value); + if (it_val != groupMap.end()) { + m_value = it_val->second; + } + } + + Type* cloneForForwardResolutionConcrete() { + return new DictType(); + } + + void postInitializeConcrete() { + m_size = sizeof(void*); + m_is_default_constructible = true; + m_bytes_per_key = m_key->bytecount(); + m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); + } + + static DictType* Make(Type* key, Type* value); // hand 'visitor' each an instance_ptr for // each value. if it returns 'false', exit early. diff --git a/typed_python/SetType.cpp b/typed_python/SetType.cpp index 6671e8fe3..d5fb560e0 100644 --- a/typed_python/SetType.cpp +++ b/typed_python/SetType.cpp @@ -3,18 +3,26 @@ #include -SetType* SetType::Make(Type* eltype, SetType* knownType) { - PyEnsureGilAcquired getTheGil; +SetType* SetType::Make(Type* eltype) { + if (eltype->isForwardDefined()) { + return new SetType(eltype); + } - static std::map m; + PyEnsureGilAcquired getTheGil; - auto it = m.find(eltype); + static std::map memo; - if (it == m.end()) { - it = m.insert(std::make_pair(eltype, knownType ? knownType : new SetType(eltype))).first; + auto it = memo.find(eltype); + if (it != memo.end()) { + return it->second; } - return it->second; + SetType* res = new SetType(eltype); + SetType* concrete = (SetType*)res->forwardResolvesTo(); + + memo[eltype] = concrete; + + return concrete; } bool SetType::discard(instance_ptr self, instance_ptr key) { @@ -65,22 +73,6 @@ int64_t SetType::size(instance_ptr self) const { return record.hash_table_count; } -bool SetType::_updateAfterForwardTypesChanged() { - std::string name = "Set(" + m_key_type->name(true) + ")"; - m_size = sizeof(void*); - m_is_default_constructible = true; - m_bytes_per_el = m_key_type->bytecount(); - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool anyChanged = name != m_name; - m_name = name; - m_stripped_name = ""; - return anyChanged; -} - int64_t SetType::refcount(instance_ptr self) const { hash_table_layout& record = **(hash_table_layout**)self; return record.refcount; diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index f8e73e88c..60fe10785 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -27,13 +27,18 @@ PyDoc_STRVAR(Set_doc, ); class SetType : public Type { - public: + SetType() : Type(TypeCategory::catSet) + { + m_doc = Set_doc; + m_needs_post_init = true; + } +public: SetType(Type* eltype) : Type(TypeCategory::catSet) , m_key_type(eltype) { m_doc = Set_doc; - endOfConstructorInitialization(); + m_is_forward_defined = true; } template @@ -41,16 +46,39 @@ class SetType : public Type { visitor(m_key_type); } - void _updateTypeMemosAfterForwardResolution() { - SetType::Make(m_key_type, this); - } - template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); v.visitTopo(m_key_type); } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "Set(" + + m_key_type->computeRecursiveName(typeStack) + + ")"; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_key_type = ((SetType*)forwardDefinitionOfSelf)->m_key_type; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + auto it_key = groupMap.find(m_key_type); + if (it_key != groupMap.end()) { + m_key_type = it_key->second; + } + } + + Type* cloneForForwardResolutionConcrete() { + return new SetType(); + } + + void postInitializeConcrete() { + m_size = sizeof(void*); + m_is_default_constructible = true; + m_bytes_per_el = m_key_type->bytecount(); + } + instance_ptr insertKey(instance_ptr self, instance_ptr key); instance_ptr lookupKey(instance_ptr self, instance_ptr key) const; bool discard(instance_ptr self, instance_ptr key); @@ -60,7 +88,6 @@ class SetType : public Type { void copy_constructor(instance_ptr self, instance_ptr other); void repr(instance_ptr self, ReprAccumulator& stream, bool isStr); int64_t refcount(instance_ptr self) const; - bool _updateAfterForwardTypesChanged(); int64_t size(instance_ptr self) const; bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions = false); @@ -87,7 +114,7 @@ class SetType : public Type { } } -void deepcopyConcrete( + void deepcopyConcrete( instance_ptr dest, instance_ptr src, DeepcopyContext& context @@ -233,10 +260,10 @@ void deepcopyConcrete( } } - public: - static SetType* Make(Type* eltype, SetType* knownType=nullptr); +public: + static SetType* Make(Type* eltype); - private: +private: Type* m_key_type; size_t m_bytes_per_el; bool subset(instance_ptr left, instance_ptr right); diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 7989f8999..50c7c2743 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -2,14 +2,16 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, - Tuple, NamedTuple, PointerTo, RefTo + Tuple, NamedTuple, PointerTo, RefTo, Dict, Set ) def test_nonforward_definition(): assert not isForwardDefined(OneOf(int, float)) assert not isForwardDefined(ListOf(int)) + assert not isForwardDefined(Dict(int, int)) assert not isForwardDefined(TupleOf(int)) + assert not isForwardDefined(Set(int)) assert not isForwardDefined(Tuple(int, int)) assert not isForwardDefined(PointerTo(int)) assert not isForwardDefined(RefTo(int)) @@ -22,9 +24,11 @@ def test_forward_definition(): assert issubclass(F, Forward) assert isForwardDefined(F) + assert isForwardDefined(Dict(int, F)) assert isForwardDefined(OneOf(int, F)) assert isForwardDefined(ListOf(F)) assert isForwardDefined(TupleOf(F)) + assert isForwardDefined(Set(F)) assert isForwardDefined(Tuple(int, F)) assert isForwardDefined(PointerTo(F)) assert isForwardDefined(RefTo(F)) @@ -127,3 +131,23 @@ def makeP(): assert makeP() is makeP() assert makeP().__name__ == 'RefTo(^0)' + + +def test_recursive_dict(): + def makeP(): + F = Forward() + F.define(Dict(int, F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'Dict(int, ^0)' + + +def test_recursive_set(): + def makeP(): + F = Forward() + F.define(Set(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'Set(^0)' From 9e8ecc2fbce58e04ff8402f9e4ced3734075c94d Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 26 May 2023 23:09:33 +0000 Subject: [PATCH 10/83] classes and stuff --- typed_python/AlternativeMatcherType.hpp | 61 ++++++---- typed_python/AlternativeType.cpp | 79 +++++------- typed_python/AlternativeType.hpp | 91 ++++++++++++-- typed_python/BoundMethodType.hpp | 72 ++++++----- typed_python/ClassType.cpp | 10 -- typed_python/ClassType.hpp | 72 ++++++++--- typed_python/ConcreteAlternativeType.cpp | 66 +++------- typed_python/ConcreteAlternativeType.hpp | 51 +++++++- typed_python/FunctionType.hpp | 114 ++++++++++++++---- typed_python/HeldClassType.cpp | 45 +++---- typed_python/HeldClassType.hpp | 110 ++++++++++++----- .../ModuleRepresentationCopyContext.hpp | 5 +- typed_python/OneOfType.cpp | 6 - typed_python/PythonObjectOfTypeType.hpp | 26 ++-- ...onSerializationContext_deserialization.cpp | 4 +- typed_python/SetType.hpp | 5 +- typed_python/SubclassOfType.cpp | 51 +++----- typed_python/SubclassOfType.hpp | 39 ++++-- typed_python/TupleOrListOfType.cpp | 8 -- typed_python/TupleOrListOfType.hpp | 2 +- typed_python/Type.hpp | 15 ++- typed_python/__init__.py | 16 +-- typed_python/_types.cpp | 6 +- typed_python/internals.py | 2 +- typed_python/type_construction_test.py | 28 ++++- 25 files changed, 596 insertions(+), 388 deletions(-) diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index e523c3ce2..b03d7b7f4 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -21,9 +21,14 @@ // Models an alternatives '.matches' result class AlternativeMatcher : public Type { + AlternativeMatcher() : Type(TypeCategory::catAlternativeMatcher) + { + m_needs_post_init = true; + } public: AlternativeMatcher(Type* inAlternative) : Type(TypeCategory::catAlternativeMatcher) { + m_is_forward_defined = true; m_is_default_constructible = false; m_alternative = inAlternative; m_size = inAlternative->bytecount(); @@ -32,6 +37,12 @@ class AlternativeMatcher : public Type { endOfConstructorInitialization(); // finish initializing the type object. } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "AlternativeMatcher(" + + m_alternative->computeRecursiveName(typeStack) + + ")"; + } + template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -48,42 +59,42 @@ class AlternativeMatcher : public Type { visitor(m_alternative); } - bool _updateAfterForwardTypesChanged() { - bool anyChanged = false; - - std::string name = "AlternativeMatcher(" + m_alternative->name(true) + ")"; - size_t size = m_alternative->bytecount(); - - anyChanged = ( - name != m_name || - size != m_size - ); + void postInitializeConcrete() { + m_size = m_alternative->bytecount(); + } - m_name = name; - m_stripped_name = ""; - m_size = size; + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_alternative = ((AlternativeMatcher*)forwardDefinitionOfSelf)->m_alternative; + } - return anyChanged; + Type* cloneForForwardResolutionConcrete() { + return new AlternativeMatcher(); } - void _updateTypeMemosAfterForwardResolution() { - AlternativeMatcher::Make(m_alternative, this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_alternative, groupMap); } - static AlternativeMatcher* Make(Type* alt, AlternativeMatcher* knownType=nullptr) { - PyEnsureGilAcquired getTheGil; + static AlternativeMatcher* Make(Type* alt) { + if (alt->isForwardDefined()) { + return new AlternativeMatcher(alt); + } - static std::map m; + PyEnsureGilAcquired getTheGil; - auto it = m.find(alt); + static std::map memo; - if (it == m.end()) { - it = m.insert( - std::make_pair(alt, knownType ? knownType : new AlternativeMatcher(alt)) - ).first; + auto it = memo.find(alt); + if (it != memo.end()) { + return it->second; } - return it->second; + AlternativeMatcher* res = new AlternativeMatcher(alt); + AlternativeMatcher* concrete = (AlternativeMatcher*)res->forwardResolvesTo(); + + memo[alt] = concrete; + + return concrete; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/AlternativeType.cpp b/typed_python/AlternativeType.cpp index ab255a31f..e158a4e6d 100644 --- a/typed_python/AlternativeType.cpp +++ b/typed_python/AlternativeType.cpp @@ -50,51 +50,6 @@ int64_t Alternative::refcount(instance_ptr i) const { return ((layout**)i)[0]->refcount; } -bool Alternative::_updateAfterForwardTypesChanged() { - m_arg_positions.clear(); - m_default_construction_type = nullptr; - - bool is_default_constructible = false; - bool all_alternatives_empty = true; - int default_construction_ix = 0; - - for (auto& subtype_pair: m_subtypes) { - if (subtype_pair.second->bytecount() > 0) { - all_alternatives_empty = false; - } - - if (m_arg_positions.find(subtype_pair.first) != m_arg_positions.end()) { - throw std::runtime_error("Can't create an alternative with " + - subtype_pair.first + " defined twice."); - } - - size_t argPosition = m_arg_positions.size(); - - m_arg_positions[subtype_pair.first] = argPosition; - - if (subtype_pair.second->is_default_constructible() && !is_default_constructible) { - is_default_constructible = true; - default_construction_ix = m_arg_positions[subtype_pair.first]; - } - } - - size_t size = (all_alternatives_empty ? 1 : sizeof(void*)); - - bool anyChanged = ( - size != m_size || - m_default_construction_ix != default_construction_ix || - m_all_alternatives_empty != all_alternatives_empty || - m_is_default_constructible != is_default_constructible - ); - - m_size = size; - m_default_construction_ix = default_construction_ix; - m_all_alternatives_empty = all_alternatives_empty; - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - bool Alternative::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (m_all_alternatives_empty) { if (*(uint8_t*)left < *(uint8_t*)right) { @@ -236,12 +191,34 @@ void Alternative::assign(instance_ptr self, instance_ptr other) { // static Alternative* Alternative::Make( - std::string name, - std::string moduleName, - const std::vector >& types, - const std::map& methods //methods preclude us from being in the memo - ) { - return new Alternative(name, moduleName, types, methods); + std::string name, + std::string moduleName, + const std::vector >& types, + const std::map& methods //methods preclude us from being in the memo +) { + bool anyForward = false; + + for (auto nameAndTup: types) { + if (nameAndTup.second->isForwardDefined()) { + anyForward = true; + } + } + + for (auto nameAndMeth: methods) { + if (nameAndMeth.second->isForwardDefined()) { + anyForward = true; + } + } + + Alternative* res = new Alternative(name, moduleName, types, methods); + + if (anyForward) { + return res; + } + + Alternative* concrete = (Alternative*)res->forwardResolvesTo(); + + return concrete; } Type* Alternative::concreteSubtype(size_t which) { diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index f2649d247..81d0288ea 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -77,6 +77,12 @@ PyDoc_STRVAR(Alternative_doc, ); class Alternative : public Type { + Alternative() : Type(TypeCategory::catAlternative) + { + m_doc = Alternative_doc; + m_needs_post_init = true; + } + public: class layout { public: @@ -100,6 +106,8 @@ class Alternative : public Type { m_methods(methods), m_hasGetAttributeMagicMethod(false) { + m_is_forward_defined = true; + m_name = name; m_moduleName = moduleName; m_is_simple = false; @@ -110,12 +118,75 @@ class Alternative : public Type { } m_doc = Alternative_doc; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + Alternative* selfT = (Alternative*)forwardDefinitionOfSelf; + + m_name = selfT->m_name; + m_moduleName = selfT->m_moduleName; + m_is_simple = selfT->m_is_simple; + m_hasGetAttributeMagicMethod = selfT->m_hasGetAttributeMagicMethod; + m_methods = selfT->m_methods; + m_subtypes = selfT->m_subtypes; + m_default_construction_ix = selfT->m_default_construction_ix; + m_default_construction_type = selfT->m_default_construction_type; + } + + Type* cloneForForwardResolutionConcrete() { + return new Alternative(); + } + + void postInitializeConcrete() { + m_arg_positions.clear(); + m_default_construction_type = nullptr; + + bool is_default_constructible = false; + bool all_alternatives_empty = true; + int default_construction_ix = 0; + + for (auto& subtype_pair: m_subtypes) { + if (subtype_pair.second->bytecount() > 0) { + all_alternatives_empty = false; + } - // call this _first_, so that our core properties, (which we know) are created - // before we walk down to the ConcreteAlternatives - _updateAfterForwardTypesChanged(); + if (m_arg_positions.find(subtype_pair.first) != m_arg_positions.end()) { + throw std::runtime_error("Can't create an alternative with " + + subtype_pair.first + " defined twice."); + } - endOfConstructorInitialization(); // finish initializing the type object. + size_t argPosition = m_arg_positions.size(); + + m_arg_positions[subtype_pair.first] = argPosition; + + if (subtype_pair.second->is_default_constructible() && !is_default_constructible) { + is_default_constructible = true; + default_construction_ix = m_arg_positions[subtype_pair.first]; + } + } + + size_t size = (all_alternatives_empty ? 1 : sizeof(void*)); + + m_size = size; + m_default_construction_ix = default_construction_ix; + m_all_alternatives_empty = all_alternatives_empty; + m_is_default_constructible = is_default_constructible; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + for (auto& nameAndSub: m_subtypes) { + updateTypeRefFromGroupMap(nameAndSub.second, groupMap); + } + + for (auto& sub: m_subtypes_concrete) { + updateTypeRefFromGroupMap(sub, groupMap); + } + + updateTypeRefFromGroupMap(m_default_construction_type, groupMap); + + for (auto& nameAndSub: m_methods) { + updateTypeRefFromGroupMap(nameAndSub.second, groupMap); + } } std::string nameWithModuleConcrete() { @@ -163,8 +234,6 @@ class Alternative : public Type { } } - bool _updateAfterForwardTypesChanged(); - template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -304,11 +373,11 @@ class Alternative : public Type { void assign(instance_ptr self, instance_ptr other); static Alternative* Make( - std::string name, - std::string moduleName, - const std::vector >& types, - const std::map& methods //methods preclude us from being in the memo - ); + std::string name, + std::string moduleName, + const std::vector >& types, + const std::map& methods + ); Alternative* renamed(std::string newName) { return Make(newName, m_moduleName, m_subtypes, m_methods); diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index 8dd5a56ca..d1514c723 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -20,16 +20,18 @@ #include "ReprAccumulator.hpp" class BoundMethod : public Type { + BoundMethod() : Type(TypeCategory::catBoundMethod) + { + m_needs_post_init = true; + } public: BoundMethod(Type* inFirstArg, std::string funcName) : Type(TypeCategory::catBoundMethod) { + m_is_forward_defined = true; + m_funcName = funcName; - m_is_default_constructible = false; m_first_arg = inFirstArg; m_size = inFirstArg->bytecount(); - m_is_simple = false; - - endOfConstructorInitialization(); // finish initializing the type object. } template @@ -49,48 +51,54 @@ class BoundMethod : public Type { visitor(m_first_arg); } - bool _updateAfterForwardTypesChanged() { - bool anyChanged = false; - - // note that you can't have '.' in the name of a type, so we use '::'. - // otherwise, the __name__ attribute of the type gets cut off at the last '.' and - // looks like a memory corruption issue. It also prevents you from knowing what - // type you're looking at. - std::string name = "BoundMethod(" + m_first_arg->name(true) + ", " + m_funcName + ")"; - size_t size = m_first_arg->bytecount(); + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "BoundMethod(" + + m_first_arg->computeRecursiveName(typeStack) + + ", " + m_funcName + + ")"; + } - anyChanged = ( - name != m_name || - size != m_size - ); + void postInitializeConcrete() { + m_is_simple = false; + m_size = m_first_arg->bytecount(); + m_is_default_constructible = false; + } - m_name = name; - m_stripped_name = ""; - m_size = size; + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_first_arg = ((BoundMethod*)forwardDefinitionOfSelf)->m_first_arg; + m_funcName = ((BoundMethod*)forwardDefinitionOfSelf)->m_funcName; + } - return anyChanged; + Type* cloneForForwardResolutionConcrete() { + return new BoundMethod(); } - void _updateTypeMemosAfterForwardResolution() { - BoundMethod::Make(m_first_arg, m_funcName, this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_first_arg, groupMap); } - static BoundMethod* Make(Type* c, std::string funcName, BoundMethod* knownType=nullptr) { + static BoundMethod* Make(Type* firstArg, std::string funcName) { + if (firstArg->isForwardDefined()) { + return new BoundMethod(firstArg, funcName); + } + PyEnsureGilAcquired getTheGil; typedef std::pair keytype; - static std::map m; - - auto it = m.find(keytype(c, funcName)); + static std::map memo; - if (it == m.end()) { - it = m.insert( - std::make_pair(keytype(c, funcName), knownType ? knownType : new BoundMethod(c, funcName)) - ).first; + auto it = memo.find(keytype(firstArg, funcName)); + if (it != memo.end()) { + return it->second; } - return it->second; + BoundMethod* res = new BoundMethod(firstArg, funcName); + BoundMethod* concrete = (BoundMethod*)res->forwardResolvesTo(); + + memo[keytype(firstArg, funcName)] = concrete; + + return concrete; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/ClassType.cpp b/typed_python/ClassType.cpp index 9ff6f8122..a65290d86 100644 --- a/typed_python/ClassType.cpp +++ b/typed_python/ClassType.cpp @@ -26,16 +26,6 @@ bool Class::isBinaryCompatibleWithConcrete(Type* other) { return m_heldClass->isBinaryCompatibleWith(otherO->m_heldClass); } -bool Class::_updateAfterForwardTypesChanged() { - bool is_default_constructible = m_heldClass->is_default_constructible(); - - bool anyChanged = m_is_default_constructible != is_default_constructible; - - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - instance_ptr Class::eltPtr(instance_ptr self, int64_t ix) const { layout& l = *instanceToLayout(self); return m_heldClass->eltPtr(l.data, ix); diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index dfa60ad98..31b437151 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -35,6 +35,13 @@ PyDoc_STRVAR(Class_doc, ); class Class : public Type { + // this is the non-forward type resolution version of this + Class() : Type(catClass) + { + m_needs_post_init = true; + m_doc = Class_doc; + } + public: class layout { public: @@ -49,15 +56,34 @@ class Class : public Type { Type(catClass), m_heldClass(inClass) { + // if the Held class is not forward, then neither are we! + m_is_forward_defined = m_heldClass->isForwardDefined(); m_size = sizeof(layout*); m_is_default_constructible = inClass->is_default_constructible(); m_name = name; m_doc = Class_doc; m_is_simple = false; + } + + void postInitializeConcrete() { + m_size = sizeof(layout*); + m_is_default_constructible = m_heldClass->is_default_constructible(); + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_heldClass = ((Class*)forwardDefinitionOfSelf)->m_heldClass; + m_name = ((Class*)forwardDefinitionOfSelf)->m_name; + m_heldClass->setClassType(this); + } - endOfConstructorInitialization(); // finish initializing the type object. + Type* cloneForForwardResolutionConcrete() { + return new Class(); + } - inClass->setClassType(this); + void updateInternalTypePointersConcrete(const std::map& groupMap) { + _visitReferencedTypes([&](Type*& typePtr) { + updateTypeRefFromGroupMap(typePtr, groupMap); + }); } // convert an instance of the class to an actual layout pointer. Because @@ -123,8 +149,6 @@ class Class : public Type { return actualTypeForLayout(self); } - bool _updateAfterForwardTypesChanged(); - static Class* Make( std::string inName, const std::vector& bases, @@ -134,8 +158,7 @@ class Class : public Type { const std::map& staticFunctions, const std::map& propertyFunctions, const std::map& classMembers, - const std::map& classMethods, - bool isNew + const std::map& classMethods ) { std::vector heldClassBases; @@ -144,21 +167,32 @@ class Class : public Type { heldClassBases.push_back(c->getHeldClass()); } - return new Class( + HeldClass* hc = HeldClass::Make( inName, - HeldClass::Make( - inName, - heldClassBases, - isFinal, - members, - memberFunctions, - staticFunctions, - propertyFunctions, - classMembers, - classMethods, - isNew - ) + heldClassBases, + isFinal, + members, + memberFunctions, + staticFunctions, + propertyFunctions, + classMembers, + classMethods ); + + PyEnsureGilAcquired getTheGil; + static std::map memo; + + auto it = memo.find(hc); + if (it != memo.end()) { + return it->second; + } + + Class* result = new Class(inName, hc); + hc->setClassType(result); + + memo[hc] = result; + + return result; } static const char* pyComparisonOpToMethodName(int pyComparisonOp) { diff --git a/typed_python/ConcreteAlternativeType.cpp b/typed_python/ConcreteAlternativeType.cpp index 0cf8348f9..d92f96554 100644 --- a/typed_python/ConcreteAlternativeType.cpp +++ b/typed_python/ConcreteAlternativeType.cpp @@ -32,35 +32,6 @@ bool ConcreteAlternative::isBinaryCompatibleWithConcrete(Type* other) { return false; } -bool ConcreteAlternative::_updateAfterForwardTypesChanged() { - if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { - throw std::runtime_error( - "invalid alternative index: " + - format(m_which) + " not in [0," + - format(m_alternative->subtypes().size()) + ")" - ); - } - - m_base = m_alternative; - - std::string name = m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; - size_t size = m_alternative->bytecount(); - bool is_default_constructible = m_alternative->subtypes()[m_which].second->is_default_constructible(); - - bool anyChanged = ( - m_name != name || - m_size != size || - m_is_default_constructible != is_default_constructible - ); - - m_name = name; - m_stripped_name = ""; - m_size = size; - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - void ConcreteAlternative::constructor(instance_ptr self) { assertForwardsResolvedSufficientlyToInstantiate(); @@ -74,31 +45,26 @@ void ConcreteAlternative::constructor(instance_ptr self) { } // static -ConcreteAlternative* ConcreteAlternative::Make(Alternative* alt, int64_t which, ConcreteAlternative* knownType) { +ConcreteAlternative* ConcreteAlternative::Make(Alternative* alt, int64_t which) { + if (alt->isForwardDefined()) { + return new ConcreteAlternative(alt, which); + } + PyEnsureGilAcquired getTheGil; typedef std::pair keytype; - static std::map m; - - auto it = m.find(keytype(alt ,which)); - - if (it == m.end()) { - if (which < 0 || which >= alt->subtypes().size()) { - throw std::runtime_error( - "invalid alternative index: " + - format(which) + " not in [0," + - format(alt->subtypes().size()) + ")" - ); - } - - it = m.insert( - std::make_pair( - keytype(alt,which), - knownType ? knownType : new ConcreteAlternative(alt,which) - ) - ).first; + static std::map memo; + + auto it = memo.find(keytype(alt, which)); + + if (it != memo.end()) { + return it->second; } - return it->second; + ConcreteAlternative* res = new ConcreteAlternative(alt, which); + ConcreteAlternative* concrete = (ConcreteAlternative*)res->forwardResolvesTo(); + + memo[keytype(alt, which)] = concrete; + return concrete; } diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index c15cac2f9..6eb64b8f1 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -20,6 +20,14 @@ #include "AlternativeType.hpp" class ConcreteAlternative : public Type { + ConcreteAlternative() : + Type(TypeCategory::catConcreteAlternative), + m_which(0), + m_alternative(nullptr) + { + m_needs_post_init = true; + } + public: typedef Alternative::layout layout; @@ -28,7 +36,42 @@ class ConcreteAlternative : public Type { m_alternative(m_alternative), m_which(which) { - endOfConstructorInitialization(); // finish initializing the type object. + m_is_forward_defined = true; + } + + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_alternative = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_alternative; + m_which = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_which; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_alternative, groupMap); + updateTypeRefFromGroupMap(m_base, groupMap); + } + + Type* cloneForForwardResolutionConcrete() { + return new ConcreteAlternative(); + } + + void postInitializeConcrete() { + if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { + throw std::runtime_error( + "invalid alternative index: " + + format(m_which) + " not in [0," + + format(m_alternative->subtypes().size()) + ")" + ); + } + + m_base = m_alternative; + size_t size = m_alternative->bytecount(); + bool is_default_constructible = m_alternative->subtypes()[m_which].second->is_default_constructible(); + + m_size = size; + m_is_default_constructible = is_default_constructible; } std::string nameWithModuleConcrete() { @@ -55,10 +98,6 @@ class ConcreteAlternative : public Type { return m_alternative->deepBytecount(instance, alreadyVisited, outSlabs); } - void _updateTypeMemosAfterForwardResolution() { - ConcreteAlternative::Make(m_alternative, m_which, this); - } - bool isBinaryCompatibleWithConcrete(Type* other); template @@ -152,7 +191,7 @@ class ConcreteAlternative : public Type { return m_alternative->eltPtr(self); } - static ConcreteAlternative* Make(Alternative* alt, int64_t which, ConcreteAlternative* knownType=nullptr); + static ConcreteAlternative* Make(Alternative* alt, int64_t which); Type* elementType() const { return m_alternative->subtypes()[m_which].second; diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 239733a73..da2296346 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1516,6 +1516,13 @@ class Function : public Type { size_t mMaxPositionalArgs; }; + Function() : Type(catFunction) + { + m_needs_post_init = true; + m_is_simple = false; + m_doc = Function_doc; + } + Function(std::string inName, std::string qualname, std::string moduleName, @@ -1530,15 +1537,12 @@ class Function : public Type { mIsNocompile(isNocompile), mRootName(inName), mQualname(qualname), - mModulename(moduleName) + mModulename(moduleName), + mClosureType(closureType) { + m_is_forward_defined = true; m_is_simple = false; m_doc = Function_doc; - - mClosureType = closureType; - - _updateAfterForwardTypesChanged(); - endOfConstructorInitialization(); // finish initializing the type object. } std::string moduleNameConcrete() { @@ -1557,15 +1561,35 @@ class Function : public Type { return mModulename + "." + mRootName; } - bool _updateAfterForwardTypesChanged() { - m_name = mRootName; - m_stripped_name = ""; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return mRootName; + } + void postInitializeConcrete() { m_size = mClosureType->bytecount(); - m_is_default_constructible = mClosureType->is_default_constructible(); + } - return false; + void initializeFromConcrete(Type* forwardDef) { + Function* fwdFunc = (Function*)forwardDef; + + mClosureType = fwdFunc->mClosureType; + mOverloads = fwdFunc->mOverloads; + mIsEntrypoint = fwdFunc->mIsEntrypoint; + mIsNocompile = fwdFunc->mIsNocompile; + mRootName = fwdFunc->mRootName; + mQualname = fwdFunc->mQualname; + mModulename = fwdFunc->mModulename; + } + + Type* cloneForForwardResolutionConcrete() { + return new Function(); + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + _visitReferencedTypes([&](Type*& typePtr) { + updateTypeRefFromGroupMap(typePtr, groupMap); + }); } template @@ -1590,22 +1614,68 @@ class Function : public Type { } } - static Function* Make(std::string inName, std::string qualname, std::string moduleName, const std::vector& overloads, Type* closureType, bool isEntrypoint, bool isNocompile) { + static Function* Make( + std::string inName, + std::string qualname, + std::string moduleName, + std::vector overloads, + Type* closureType, + bool isEntrypoint, + bool isNocompile + ) { + bool anyFwd = false; + + if (closureType->isForwardDefined()) { + anyFwd = true; + } + + for (auto& o: overloads) { + o._visitReferencedTypes([&](Type* t) { + if (t->isForwardDefined()) { + anyFwd = true; + } + }); + } + + if (anyFwd) { + return new Function( + inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile + ); + } + PyEnsureGilAcquired getTheGil; - typedef std::tuple, Type*, bool, bool> keytype; + typedef std::tuple< + const std::string, + const std::string, + const std::string, + const std::vector, + Type*, + bool, + bool + > keytype; + + // allocate and leak the memo or else we will crash at process unload time + static std::map *memo = new std::map(); + + keytype key( + inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile + ); - static std::map *m = new std::map(); + auto it = memo->find(key); - auto it = m->find(keytype(inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile)); - if (it == m->end()) { - it = m->insert(std::pair( - keytype(inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile), - new Function(inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile) - )).first; + if (it != memo->end()) { + return it->second; } - return it->second; + Function* res = new Function( + inName, qualname, moduleName, overloads, closureType, isEntrypoint, isNocompile + ); + + Function* concrete = (Function*)res->forwardResolvesTo(); + + (*memo)[key] = concrete; + return concrete; } template @@ -1658,8 +1728,6 @@ class Function : public Type { return mClosureType->cmp(left, right, pyComparisonOp, suppressExceptions); } - - void deepcopyConcrete( instance_ptr dest, instance_ptr src, diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index d522cd098..1c5033e82 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -66,26 +66,6 @@ void HeldClass::updateBytesOfInitBits() { } } -bool HeldClass::_updateAfterForwardTypesChanged() { - m_byte_offsets.clear(); - - updateBytesOfInitBits(); - - size_t size = mBytesOfInitializationBits; - - for (auto t: m_members) { - m_byte_offsets.push_back(size); - size += t.getType()->bytecount(); - } - - bool anyChanged = size != m_size; - - m_is_default_constructible = m_memberFunctions.find("__init__") == m_memberFunctions.end(); - m_size = size; - - return anyChanged; -} - bool HeldClass::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (hasAnyComparisonOperators()) { auto it = getMemberFunctions().find(Class::pyComparisonOpToMethodName(pyComparisonOp)); @@ -441,9 +421,9 @@ HeldClass* HeldClass::Make( const std::map& memberFunctions, const std::map& staticFunctions, const std::map& propertyFunctions, + // TODO: are we accidentally leaking forwards out of this? Probably. const std::map& classMembers, - const std::map& classMethods, - bool isNew + const std::map& classMethods ) { //we only allow one base class to have members because we want native code to be //able to just find those values in subclasses without hitting the vtable. @@ -505,16 +485,23 @@ HeldClass* HeldClass::Make( staticFunctions, propertyFunctions, classMembers, - classMethods, - isNew + classMethods ); - // we do these outside of the constructor so that if they throw we - // don't destroy the HeldClass type object (and just leak it instead) because - // we need to ensure we never delete Type objects. - result->initializeMRO(); + // check if the forward has any references to forward types. If not, we can + // (and should) resolve it immediately, since our contract is that we only produce forwards + // if we refer to forwards. + bool anyForward = false; - result->endOfConstructorInitialization(); + result->_visitReferencedTypes( + [&](Type* t) { if (t->isForwardDefined()) { anyForward = true; } } + ); + + if (!anyForward) { + return (HeldClass*)result->forwardResolvesTo(); + } + // we have a forward ref - we have to go through the normal duplicate-and-intern forward + // construction process for this to work. return result; } diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 18ba217a9..a52a48c1d 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -339,21 +339,34 @@ class VTable { //a class held directly inside of another object class HeldClass : public Type { + HeldClass() : + Type(catHeldClass), + m_vtable(new VTable(this)), + m_classType(nullptr), + m_is_final(false), + m_hasComparisonOperators(false), + m_hasGetAttributeMagicMethod(false), + m_hasGetAttrMagicMethod(false), + m_hasSetAttrMagicMethod(false), + m_hasDelAttrMagicMethod(false), + m_hasDelMagicMethod(false), + m_hasConvertFromMagicMethod(false), + m_refToType(nullptr) + { + m_needs_post_init = true; + } + public: HeldClass(std::string inName, - const std::vector& baseClasses, - bool isFinal, - const std::vector& members, - const std::map& memberFunctions, - const std::map& staticFunctions, - const std::map& propertyFunctions, - const std::map& classMembers, - const std::map& classMethods, - // set to True if this is the first time the class is being created - // and we need to make copies of all the function objects so that - // they know who their methods are. - bool isNew - ) : + const std::vector& baseClasses, + bool isFinal, + const std::vector& members, + const std::map& memberFunctions, + const std::map& staticFunctions, + const std::map& propertyFunctions, + const std::map& classMembers, + const std::map& classMethods + ) : Type(catHeldClass), m_vtable(new VTable(this)), m_bases(baseClasses), @@ -374,23 +387,66 @@ class HeldClass : public Type { m_hasConvertFromMagicMethod(false), m_refToType(nullptr) { + m_is_forward_defined = true; + m_name = inName; - if (isNew) { - for (auto& nameAndF: m_own_memberFunctions) { - nameAndF.second = nameAndF.second->withMethodOf(this); - } + for (auto& nameAndF: m_own_memberFunctions) { + nameAndF.second = nameAndF.second->withMethodOf(this); + } - for (auto& nameAndF: m_own_classMethods) { - nameAndF.second = nameAndF.second->withMethodOf(this); - } + for (auto& nameAndF: m_own_classMethods) { + nameAndF.second = nameAndF.second->withMethodOf(this); + } - for (auto& nameAndF: m_own_staticFunctions) { - nameAndF.second = nameAndF.second->withMethodOf(this); - } + for (auto& nameAndF: m_own_staticFunctions) { + nameAndF.second = nameAndF.second->withMethodOf(this); } } + void postInitializeConcrete() { + m_byte_offsets.clear(); + + updateBytesOfInitBits(); + + size_t size = mBytesOfInitializationBits; + + for (auto t: m_members) { + m_byte_offsets.push_back(size); + size += t.getType()->bytecount(); + } + + m_is_default_constructible = ( + m_memberFunctions.find("__init__") == m_memberFunctions.end() + ); + m_size = size; + + initializeMRO(); + } + + void initializeFromConcrete(Type* forwardDef) { + HeldClass* fwdCls = (HeldClass*)forwardDef; + + m_bases = fwdCls->m_bases; + m_is_final = fwdCls->m_is_final; + m_own_members = fwdCls->m_own_members; + m_own_memberFunctions = fwdCls->m_own_memberFunctions; + m_own_staticFunctions = fwdCls->m_own_staticFunctions; + m_own_propertyFunctions = fwdCls->m_own_propertyFunctions; + m_own_classMembers = fwdCls->m_own_classMembers; + m_own_classMethods = fwdCls->m_own_classMethods; + } + + Type* cloneForForwardResolutionConcrete() { + return new HeldClass(); + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + _visitReferencedTypes([&](Type*& typePtr) { + updateTypeRefFromGroupMap(typePtr, groupMap); + }); + } + template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -502,8 +558,6 @@ class HeldClass : public Type { } } - bool _updateAfterForwardTypesChanged(); - static HeldClass* Make( std::string inName, const std::vector& bases, @@ -513,8 +567,7 @@ class HeldClass : public Type { const std::map& staticFunctions, const std::map& propertyFunctions, const std::map& classMembers, - const std::map& classMethods, - bool isNew + const std::map& classMethods ); // this gets called by Class. These types are always produced in pairs. @@ -549,8 +602,7 @@ class HeldClass : public Type { m_own_staticFunctions, m_own_propertyFunctions, m_own_classMembers, - m_own_classMethods, - true + m_own_classMethods ); } diff --git a/typed_python/ModuleRepresentationCopyContext.hpp b/typed_python/ModuleRepresentationCopyContext.hpp index a560c65ab..5a102fdbe 100644 --- a/typed_python/ModuleRepresentationCopyContext.hpp +++ b/typed_python/ModuleRepresentationCopyContext.hpp @@ -306,10 +306,7 @@ class ModuleRepresentationCopyContext { staticFunctions, propertyFunctions, classMembers, - classMethods, - // we want to ensure the class has appropriate - // methodOf for all of its function types - true + classMethods ); forwardC->define(outC); diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index fac7beb4b..b3f250c36 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -182,10 +182,6 @@ void OneOfType::updateInternalTypePointersConcrete( } void OneOfType::postInitializeConcrete() { - if (!m_needs_post_init) { - return; - } - for (auto t: m_types) { t->postInitialize(); } @@ -200,8 +196,6 @@ void OneOfType::postInitializeConcrete() { break; } } - - m_needs_post_init = false; } std::string OneOfType::computeRecursiveNameConcrete(TypeStack& typeStack) { diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index 18e3954f0..476339d33 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -128,7 +128,15 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { m_is_simple = false; - endOfConstructorInitialization(); // finish initializing the type object. + m_size = sizeof(layout_type*); + + int isinst = PyObject_IsInstance(Py_None, (PyObject*)mPyTypePtr); + if (isinst == -1) { + isinst = 0; + PyErr_Clear(); + } + + m_is_default_constructible = isinst != 0; } template @@ -149,21 +157,7 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { void _visitReferencedTypes(const visitor_type& visitor) { } - bool _updateAfterForwardTypesChanged() { - m_size = sizeof(layout_type*); - - int isinst = PyObject_IsInstance(Py_None, (PyObject*)mPyTypePtr); - if (isinst == -1) { - isinst = 0; - PyErr_Clear(); - } - - m_is_default_constructible = isinst != 0; - - //none of these values can ever change, so we can just return - //because we don't need to be updated again. - return false; - } + void postInitializeConcrete() {} int64_t refcount(instance_ptr self) const { return getHandlePtr(self)->refcount; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 2bc9f8efb..2d66b8dd6 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1643,9 +1643,7 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff classStatics, classPropertyFunctions, classClassMembersRaw, - classClassMethods, - // the methodOf should already be set in the class - false + classClassMethods )->getHeldClass(); } else if (category == Type::TypeCategory::catFunction) { diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index 60fe10785..221801f53 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -63,10 +63,7 @@ class SetType : public Type { } void updateInternalTypePointersConcrete(const std::map& groupMap) { - auto it_key = groupMap.find(m_key_type); - if (it_key != groupMap.end()) { - m_key_type = it_key->second; - } + updateTypeRefFromGroupMap(m_key_type, groupMap); } Type* cloneForForwardResolutionConcrete() { diff --git a/typed_python/SubclassOfType.cpp b/typed_python/SubclassOfType.cpp index fe4d8ba85..91645e89a 100644 --- a/typed_python/SubclassOfType.cpp +++ b/typed_python/SubclassOfType.cpp @@ -16,35 +16,6 @@ #include "AllTypes.hpp" -bool SubclassOfType::_updateAfterForwardTypesChanged() { - size_t size = sizeof(Type*); - std::string name = computeName(); - - if (m_is_recursive_forward) { - name = m_recursive_name; - } - - bool is_default_constructible = true; - - bool anyChanged = ( - size != m_size || - name != m_name || - is_default_constructible != m_is_default_constructible - ); - - m_size = size; - m_name = name; - m_stripped_name = ""; - - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} - -std::string SubclassOfType::computeName() const { - return "SubclassOf(" + m_subclassOf->name(true) + ")"; -} - void SubclassOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { stream << "name() + "'>"; } @@ -89,16 +60,24 @@ void SubclassOfType::assign(instance_ptr self, instance_ptr other) { } // static -SubclassOfType* SubclassOfType::Make(Type* subclassOf, SubclassOfType* knownType) { - PyEnsureGilAcquired getTheGil; +SubclassOfType* SubclassOfType::Make(Type* subclassOf) { + if (subclassOf->isForwardDefined()) { + return new SubclassOfType(subclassOf); + } - static std::map m; + PyEnsureGilAcquired getTheGil; - auto it = m.find(subclassOf); + static std::map memo; - if (it == m.end()) { - it = m.insert(std::make_pair(subclassOf, knownType ? knownType : new SubclassOfType(subclassOf))).first; + auto it = memo.find(subclassOf); + if (it != memo.end()) { + return it->second; } - return it->second; + SubclassOfType* res = new SubclassOfType(subclassOf); + SubclassOfType* concrete = (SubclassOfType*)res->forwardResolvesTo(); + + memo[subclassOf] = concrete; + + return concrete; } diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index 6e65aa19e..e400ba90f 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -25,14 +25,37 @@ PyDoc_STRVAR(SubclassOf_doc, ); class SubclassOfType : public Type { + SubclassOfType() : Type(TypeCategory::catSubclassOf) + { + m_doc = SubclassOf_doc; + m_is_default_constructible = true; + m_needs_post_init = true; + m_size = sizeof(Type*); + } public: SubclassOfType(Type* subclassOf) noexcept : Type(TypeCategory::catSubclassOf), m_subclassOf(subclassOf) { m_doc = SubclassOf_doc; + m_is_forward_defined = true; + m_is_default_constructible = true; + m_size = sizeof(Type*); + } + + void postInitializeConcrete() { + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_subclassOf = ((SubclassOfType*)forwardDefinitionOfSelf)->m_subclassOf; + } + + Type* cloneForForwardResolutionConcrete() { + return new SubclassOfType(); + } - endOfConstructorInitialization(); // finish initializing the type object. + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_subclassOf, groupMap); } template @@ -55,13 +78,15 @@ class SubclassOfType : public Type { _visitContainedTypes(visitor); } - bool _updateAfterForwardTypesChanged(); - bool isPODConcrete() { return true; } - std::string computeName() const; + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "SubclassOf(" + + m_subclassOf->computeRecursiveName(typeStack) + + ")"; + } void deepcopyConcrete( instance_ptr dest, @@ -104,11 +129,7 @@ class SubclassOfType : public Type { return m_subclassOf; } - void _updateTypeMemosAfterForwardResolution() { - SubclassOfType::Make(m_subclassOf, this); - } - - static SubclassOfType* Make(Type* subclassOf, SubclassOfType* knownType = nullptr); + static SubclassOfType* Make(Type* subclassOf); private: Type* m_subclassOf; diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 82cc47289..0455a4d57 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -164,14 +164,6 @@ void TupleOrListOfType::updateInternalTypePointersConcrete( } } - -void TupleOrListOfType::postInitializeConcrete() { - if (!m_needs_post_init) { - return; - } - m_needs_post_init = false; -} - std::string TupleOrListOfType::computeRecursiveNameConcrete(TypeStack& stack) { if (m_is_tuple) { return "TupleOf(" + m_element_type->computeRecursiveName(stack) + ")"; diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index fff681d63..899fd8f2a 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -596,7 +596,7 @@ class TupleOrListOfType : public Type { void initializeFromConcrete(Type* forwardDefinitionOfSelf); - void postInitializeConcrete(); + void postInitializeConcrete() {}; std::string computeRecursiveNameConcrete(TypeStack& typeStack); diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index ce47fa0ca..c57de30a0 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -590,6 +590,15 @@ class Type { } } + template + static void updateTypeRefFromGroupMap(T*& toUpdate, const std::map& groupMap) { + auto it = groupMap.find(toUpdate); + + if (it != groupMap.end()) { + toUpdate = (T*)it->second; + } + } + Type* getBaseType() const { return m_base; } @@ -987,7 +996,9 @@ class Type { } void postInitializeConcrete() { - throw std::runtime_error("Type " + name() + " of cat " + getTypeCategoryString() + " didn't implement postInitializeConcrete"); + throw std::runtime_error( + "Type " + name() + " of cat " + getTypeCategoryString() + " didn't implement postInitializeConcrete" + ); } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { @@ -1009,6 +1020,8 @@ class Type { return; } + m_needs_post_init = false; + this->check([&](auto& subtype) { subtype.postInitializeConcrete(); }); diff --git a/typed_python/__init__.py b/typed_python/__init__.py index a105eb056..13f3b54cd 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -28,14 +28,14 @@ # can check this version. __untyped_serialization_version__ = 5 -# from typed_python.internals import ( -# Member, Final, Function, UndefinedBehaviorException, -# makeNamedTuple, DisableCompiledCode, isCompiled, Held, -# typeKnownToCompiler, -# localVariableTypesKnownToCompiler, -# checkOneOfType, -# checkType -# ) +from typed_python.internals import ( + Member, Final, Function, UndefinedBehaviorException, + makeNamedTuple, DisableCompiledCode, isCompiled, Held, + typeKnownToCompiler, + localVariableTypesKnownToCompiler, + checkOneOfType, + checkType +) from typed_python._types import bytecount, refcount # from typed_python.module import Module # from typed_python.type_function import TypeFunction diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 0a75a5726..a960a3fc5 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -1284,11 +1284,7 @@ PyObject *MakeClassType(PyObject* nullValue, PyObject* args) { staticFuncs, propertyFuncs, clsMembers, - classMethodFuncs, - // this is the first time this class exists, - // so we need the constructor to rebuild the - // function objects with apprioriated methodOf - true + classMethodFuncs ) ) ); diff --git a/typed_python/internals.py b/typed_python/internals.py index b3894b41d..1b33393a1 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -380,7 +380,7 @@ def __instancecheck__(cls, instance): return cls in type(instance).MRO -def Function(f, assumeClosuresGlobal=False, returnTypeOverride=None): +def Function(f, returnTypeOverride=None, assumeClosuresGlobal=False): """Turn a normal python function into a 'typed_python.Function' which obeys type restrictions.""" return makeFunctionType( f.__name__, f, assumeClosuresGlobal=assumeClosuresGlobal, returnTypeOverride=returnTypeOverride diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 50c7c2743..ae50641a3 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -2,7 +2,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, - Tuple, NamedTuple, PointerTo, RefTo, Dict, Set + Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function ) @@ -16,6 +16,9 @@ def test_nonforward_definition(): assert not isForwardDefined(PointerTo(int)) assert not isForwardDefined(RefTo(int)) assert not isForwardDefined(NamedTuple(x=int, y=float)) + assert not isForwardDefined(Alternative("A", x={}, y={})) + assert not isForwardDefined(Alternative("A", x={}, y={}).x) + assert not isForwardDefined(Function(lambda x: x)) def test_forward_definition(): @@ -33,6 +36,13 @@ def test_forward_definition(): assert isForwardDefined(PointerTo(F)) assert isForwardDefined(RefTo(F)) assert isForwardDefined(NamedTuple(x=int, y=F)) + assert isForwardDefined(Alternative("A", x={'f': F})) + assert isForwardDefined(Alternative("A", x={'f': F}).x) + + func = Function(lambda x: x, F) + print(func.overloads[0].returnType) + + assert isForwardDefined(func) def test_forward_one_of(): @@ -151,3 +161,19 @@ def makeP(): assert makeP() is makeP() assert makeP().__name__ == 'Set(^0)' + + +def test_recursive_alternative(): + def makeA(): + F = Forward() + F.define( + Alternative( + "F", + A={'f': F}, + B={} + ) + ) + return resolveForwardDefinedType(F) + + assert makeA() is makeA() + assert makeA().__name__ == 'F' From 25d9d10871770fca8434a004623e79dc29b2cf15 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 30 May 2023 22:04:52 +0000 Subject: [PATCH 11/83] Ensure that forwards can actually be instantiated. --- typed_python/Type.hpp | 13 +++++++++++-- typed_python/type_construction_test.py | 4 +--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index c57de30a0..a2ea43fa3 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -665,8 +665,17 @@ class Type { } void assertForwardsResolvedSufficientlyToInstantiate() { + this->check([&](auto& subtype) { + subtype.assertForwardsResolvedSufficientlyToInstantiateConcrete(); + }); + } + + void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { if (m_is_forward_defined) { - throw std::logic_error("Type " + m_name + " is a forward"); + throw std::logic_error( + "Type " + m_name + " of cat " + + this->getTypeCategoryString() + " is a forward" + ); } } @@ -741,7 +750,7 @@ class Type { template void copy_constructor(int64_t count, const ptr_func_dest& ptrToTarget, const ptr_func_src& ptrToSrc) { - assertForwardsResolved(); + assertForwardsResolvedSufficientlyToInstantiate(); this->check([&](auto& subtype) { for (long k = 0; k < count; k++) { diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index ae50641a3..712f70533 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -40,9 +40,7 @@ def test_forward_definition(): assert isForwardDefined(Alternative("A", x={'f': F}).x) func = Function(lambda x: x, F) - print(func.overloads[0].returnType) - - assert isForwardDefined(func) + assert isForwardDefined(type(func)) def test_forward_one_of(): From 33a0c01a5d1e988cf67065f21982686cc1f222c0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 30 May 2023 22:11:16 +0000 Subject: [PATCH 12/83] More function stuff. Not clear that we should allow instantiation of forward defined functions like this but OTOH how do we enble type functions returning function instances? Perhaps we need these to behave like 'value' objects instead... --- typed_python/__init__.py | 3 ++- typed_python/type_construction_test.py | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 13f3b54cd..a94559baf 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -55,7 +55,8 @@ ModuleRepresentation, setGilReleaseThreadLoopSleepMicroseconds, isForwardDefined, - resolveForwardDefinedType + resolveForwardDefinedType, + identityHash ) import typed_python._types as _types import threading diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 712f70533..cb141d9df 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -2,7 +2,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, - Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function + Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash ) @@ -175,3 +175,27 @@ def makeA(): assert makeA() is makeA() assert makeA().__name__ == 'F' + + +def test_function_types_coalesc(): + def makeFNonforward(T): + @Function + def f(x: T): + return x + + return type(f) + + def makeFforward(T): + t = Forward("T") + + @Function + def f(x: t): + return x + + t.define(T) + + return resolveForwardDefinedType(type(f)) + + assert makeFNonforward(int) is makeFNonforward(int) + assert identityHash(makeFforward(int)) == identityHash(makeFforward(int)) + assert makeFforward(int) is makeFforward(int) From 186647021e2163cee22cb127ca9e1dfc416d9d82 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 31 May 2023 23:27:12 +0000 Subject: [PATCH 13/83] Works for Value types. --- typed_python/Instance.cpp | 34 ++++++++ typed_python/Instance.hpp | 11 +++ typed_python/ValueType.cpp | 114 +++++++++++++++++++++---- typed_python/ValueType.hpp | 29 ++++--- typed_python/type_construction_test.py | 41 ++++++++- 5 files changed, 202 insertions(+), 27 deletions(-) diff --git a/typed_python/Instance.cpp b/typed_python/Instance.cpp index 234eebfaa..27ecdb4c2 100644 --- a/typed_python/Instance.cpp +++ b/typed_python/Instance.cpp @@ -40,6 +40,25 @@ Instance Instance::create(Type*t, instance_ptr data) { t->copy_constructor(tgt, data); }); } + +Instance Instance::create(PyObject* inst) { + if (PyType_Check(inst)) { + return Instance::createAndInitialize( + PythonObjectOfType::AnyPyType(), + [&](instance_ptr i) { + PythonObjectOfType::AnyPyType()->initializeFromPyObject(i, inst); + } + ); + } + + return Instance::createAndInitialize( + PythonObjectOfType::AnyPyObject(), + [&](instance_ptr i) { + PythonObjectOfType::AnyPyObject()->initializeFromPyObject(i, inst); + } + ); +} + Instance Instance::create(Type*t) { t->assertForwardsResolvedSufficientlyToInstantiate(); @@ -147,3 +166,18 @@ typed_python_hash_type Instance::hash() const { return mLayout->type->hash(mLayout->data); } + +Type* Instance::extractType(bool includePrimitives) const { + if ( + type()->isPythonObjectOfType() && + PyType_Check( + PyObjectHandleTypeBase::getPyObj(data()) + ) + ) { + PyObject* obj = PyObjectHandleTypeBase::getPyObj(data()); + + return PyInstance::extractTypeFrom((PyTypeObject*)obj, includePrimitives); + } + + return nullptr; +} diff --git a/typed_python/Instance.hpp b/typed_python/Instance.hpp index f12b15421..fb70fd3d2 100644 --- a/typed_python/Instance.hpp +++ b/typed_python/Instance.hpp @@ -81,6 +81,8 @@ class Instance { static Instance create(Type*t, instance_ptr data); + static Instance create(PyObject* o); + static Instance create(Type*t); Instance(); @@ -94,6 +96,12 @@ class Instance { return Instance(t, initFun); } + Instance clone() const { + return Instance::createAndInitialize(type(), [&](instance_ptr newSelf) { + type()->copy_constructor(newSelf, data()); + }); + } + template Instance(Type* t, const initializer_type& initFun) : mLayout(nullptr) { if (t->isNone()) { @@ -134,6 +142,9 @@ class Instance { return mLayout ? mLayout->type : noneType; } + // if we wrap a Type* as a python object, return it. Otherwise nullptr. + Type* extractType(bool includePrimitives=false) const; + template T& cast() { return *(T*)data(); diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 3862973d1..547628d64 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -1,29 +1,90 @@ /****************************************************************************** - Copyright 2017-2023 typed_python Authors + Copyright 2017-2023 typed_python Authors - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. ******************************************************************************/ #include "AllTypes.hpp" +Value::Value() : + Type(TypeCategory::catValue), + mValueAsPyobj(nullptr) +{ + m_needs_post_init = true; + m_size = 0; + m_is_default_constructible = true; + m_doc = Value_doc; +} + Value::Value(const Instance& instance) : Type(TypeCategory::catValue), - mInstance(instance) + mInstance(instance), + mValueAsPyobj(nullptr) { m_size = 0; m_is_default_constructible = true; + m_doc = Value_doc; + mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + m_is_forward_defined = true; +} + +Type* Value::Make(Instance i) { + PyEnsureGilAcquired getTheGil; + + // if its a forward declared Type then we need to behave differently + if (i.type()->isPythonObjectOfType() && + PyType_Check( + PyObjectHandleTypeBase::getPyObj(i.data()) + ) + ) { + PyObject* obj = PyObjectHandleTypeBase::getPyObj(i.data()); + Type* t = PyInstance::extractTypeFrom((PyTypeObject*)obj, true); + + if (t) { + static std::map typeMemo; + + auto it = typeMemo.find(t); + + if (it != typeMemo.end()) { + return it->second; + } + + if (t->isForwardDefined()) { + typeMemo[t] = new Value(i); + } else { + typeMemo[t] = (Value*)(new Value(i))->forwardResolvesTo(); + } + + return typeMemo[t]; + } + } + + static std::map instanceMemo; + + auto it = instanceMemo.find(i); + + if (it == instanceMemo.end()) { + Value* resolved = (Value*)((new Value(i))->forwardResolvesTo()); + it = instanceMemo.insert(std::make_pair(i, resolved)).first; + } + + return it->second; +} + +std::string Value::computeRecursiveNameConcrete(TypeStack& typeStack) +{ if ( mInstance.type()->isPythonObjectOfType() && PyType_Check( @@ -32,7 +93,14 @@ Value::Value(const Instance& instance) : ) { PyObject* obj = PyObjectHandleTypeBase::getPyObj(mInstance.data()); - std::string name(((PyTypeObject*)obj)->tp_name); + Type* t = PyInstance::extractTypeFrom((PyTypeObject*)obj, true); + + std::string name; + if (t) { + name = t->computeRecursiveName(typeStack); + } else { + name = ((PyTypeObject*)obj)->tp_name; + } auto ix = name.rfind('.'); @@ -42,12 +110,28 @@ Value::Value(const Instance& instance) : ix += 1; } - m_name = "Value(" + name.substr(ix) + ")"; + return "Value(" + name.substr(ix) + ")"; } else { - m_name = mInstance.repr(); + return mInstance.repr(); } +} - m_doc = Value_doc; +void Value::updateInternalTypePointersConcrete( + const std::map& groupMap +) { + Type* ownType = mInstance.extractType(true); + + auto it = groupMap.find(ownType); + + if (it != groupMap.end()) { + mInstance = Instance::create( + (PyObject*)PyInstance::typeObj(it->second) + ); + mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + } +} + +void Value::initializeFromConcrete(Type* forwardDefinitionOfSelf) { + mInstance = ((Value*)forwardDefinitionOfSelf)->mInstance.clone(); mValueAsPyobj = PyInstance::extractPythonObject(mInstance); - endOfConstructorInitialization(); // finish initializing the type object. } diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index 4fa0c33f7..a0cedd206 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -39,6 +39,11 @@ class Value : public Type { if (t != mInstance.type()) { throw std::runtime_error("visitor shouldn't have changed the type of a Value"); } + + t = mInstance.extractType(); + if (t) { + visitor(t); + } } bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { @@ -97,26 +102,28 @@ class Value : public Type { return mInstance; } - // TODO: should we allow Forward-declared Value types? - static Type* Make(Instance i) { - PyEnsureGilAcquired getTheGil; + static Type* Make(Instance i); - static std::map m; + void postInitializeConcrete() {} - auto it = m.find(i); + Type* cloneForForwardResolutionConcrete() { + return new Value(); + } - if (it == m.end()) { - it = m.insert(std::make_pair(i, new Value(i))).first; - } + void initializeFromConcrete(Type* forwardDefinitionOfSelf); - return it->second; - } + std::string computeRecursiveNameConcrete(TypeStack& typeStack); - void postInitializeConcrete() {} + void updateInternalTypePointersConcrete( + const std::map& groupMap + ); private: Value(const Instance& instance); + // this is the clone-pathway + Value(); + Instance mInstance; PyObject* mValueAsPyobj; }; diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index cb141d9df..dc6a06d35 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -2,7 +2,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, - Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash + Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value ) @@ -121,6 +121,18 @@ def makeTup(): assert makeTup() is makeTup() +def test_nonrecursive_list_of_forward_memoizes(): + def makeTup(): + F = Forward() + O = ListOf(F) + F.define(int) + + return resolveForwardDefinedType(O) + + assert makeTup() is makeTup() + assert makeTup() is ListOf(int) + + def test_recursive_pointer_to(): def makeP(): F = Forward() @@ -177,6 +189,19 @@ def makeA(): assert makeA().__name__ == 'F' +def test_cannot_call_forward_function_instances(): + t = Forward("T") + + @Function + def f(x: t): + return x + + t.define(int) + + with pytest.raises(TypeError): + f(10) + + def test_function_types_coalesc(): def makeFNonforward(T): @Function @@ -199,3 +224,17 @@ def f(x: t): assert makeFNonforward(int) is makeFNonforward(int) assert identityHash(makeFforward(int)) == identityHash(makeFforward(int)) assert makeFforward(int) is makeFforward(int) + + +def test_create_value_type_with_forward(): + F = Forward("X") + T = Value(F) + F.define(int) + + assert isForwardDefined(T) + + T_resolved = resolveForwardDefinedType(T) + + assert not isForwardDefined(T_resolved) + assert identityHash(T_resolved).hex() == identityHash(Value(int)).hex() + assert T_resolved is Value(int) From 94305415e013414d7534f2400dc0de61e98433cb Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 1 Jun 2023 22:31:00 +0000 Subject: [PATCH 14/83] Ensure we can construct forward Class/HeldClass types. --- typed_python/ClassType.hpp | 32 ++++---- typed_python/HeldClassType.cpp | 8 +- typed_python/HeldClassType.hpp | 85 +++++++++---------- typed_python/OneOfType.cpp | 6 ++ typed_python/Type.hpp | 16 ++++ typed_python/TypeOrPyobj.cpp | 1 + typed_python/type_construction_test.py | 108 ++++++++++++++++++++++++- 7 files changed, 191 insertions(+), 65 deletions(-) diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index 31b437151..4e82933b4 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -73,7 +73,6 @@ class Class : public Type { void initializeFromConcrete(Type* forwardDefinitionOfSelf) { m_heldClass = ((Class*)forwardDefinitionOfSelf)->m_heldClass; m_name = ((Class*)forwardDefinitionOfSelf)->m_name; - m_heldClass->setClassType(this); } Type* cloneForForwardResolutionConcrete() { @@ -81,6 +80,8 @@ class Class : public Type { } void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_heldClass, groupMap); + _visitReferencedTypes([&](Type*& typePtr) { updateTypeRefFromGroupMap(typePtr, groupMap); }); @@ -140,9 +141,17 @@ class Class : public Type { template void _visitReferencedTypes(const visitor_type& visitor) { - Type* t = m_heldClass; - visitor(t); - assert(t == m_heldClass); + if (m_heldClass) { + Type* hc = m_heldClass; + visitor(hc); + if (hc != (Type*)m_heldClass) { + if (!hc || !hc->isHeldClass()) { + throw std::runtime_error("Can't set a Class's heldClass to a non HeldClass"); + } + + m_heldClass = (HeldClass*)hc; + } + } } Type* pickConcreteSubclassConcrete(instance_ptr self) { @@ -179,20 +188,7 @@ class Class : public Type { classMethods ); - PyEnsureGilAcquired getTheGil; - static std::map memo; - - auto it = memo.find(hc); - if (it != memo.end()) { - return it->second; - } - - Class* result = new Class(inName, hc); - hc->setClassType(result); - - memo[hc] = result; - - return result; + return hc->getClassType(); } static const char* pyComparisonOpToMethodName(int pyComparisonOp) { diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index 1c5033e82..4d0834c41 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -476,6 +476,7 @@ HeldClass* HeldClass::Make( throw PythonExceptionSet(); } + // this is a forward-defined HeldClass HeldClass* result = new HeldClass( "Held(" + inName + ")", bases, @@ -488,13 +489,18 @@ HeldClass* HeldClass::Make( classMethods ); + Class* clsType = new Class(inName, result); + + // initialize the corresonding Class type + result->setClassType(clsType); + // check if the forward has any references to forward types. If not, we can // (and should) resolve it immediately, since our contract is that we only produce forwards // if we refer to forwards. bool anyForward = false; result->_visitReferencedTypes( - [&](Type* t) { if (t->isForwardDefined()) { anyForward = true; } } + [&](Type* t) { if (t->isForwardDefined() && t != clsType) { anyForward = true; } } ); if (!anyForward) { diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index a52a48c1d..588b83aba 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -405,28 +405,17 @@ class HeldClass : public Type { } void postInitializeConcrete() { - m_byte_offsets.clear(); - - updateBytesOfInitBits(); - - size_t size = mBytesOfInitializationBits; - - for (auto t: m_members) { - m_byte_offsets.push_back(size); - size += t.getType()->bytecount(); + for (auto b: m_bases) { + b->postInitialize(); } - m_is_default_constructible = ( - m_memberFunctions.find("__init__") == m_memberFunctions.end() - ); - m_size = size; - initializeMRO(); } void initializeFromConcrete(Type* forwardDef) { HeldClass* fwdCls = (HeldClass*)forwardDef; + m_classType = fwdCls->m_classType; m_bases = fwdCls->m_bases; m_is_final = fwdCls->m_is_final; m_own_members = fwdCls->m_own_members; @@ -504,57 +493,42 @@ class HeldClass : public Type { template void _visitReferencedTypes(const visitor_type& visitor) { - for (auto& b: m_bases) { - Type* baseT = b; - visitor(baseT); - if (b != baseT) { - throw std::runtime_error("Somehow, we modified the base type of a HeldClass?"); - } + if (m_classType) { + adaptTypeVisitor(visitor, m_classType); + } + + for (long k = 0; k < m_bases.size(); k++) { + adaptTypeVisitor(visitor, m_bases[k]); } + for (auto& o: m_own_members) { // this is expected to actually modify the type // if its a forward. - visitor(o.getType()); + adaptTypeVisitor(visitor, o.getType()); } for (auto& o: m_own_memberFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_own_staticFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_own_classMethods) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_own_propertyFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_members) { - // this is expected to actually modify the type - // if its a forward. - visitor(o.getType()); + adaptTypeVisitor(visitor, o.getType()); } for (auto& o: m_memberFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_staticFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } for (auto& o: m_propertyFunctions) { - Type* t = o.second; - visitor(t); - assert(t == o.second); + adaptTypeVisitor(visitor, o.second); } } @@ -941,6 +915,10 @@ class HeldClass : public Type { } void initializeMRO() { + if (m_is_forward_defined) { + throw std::runtime_error("Makes no sense to initializeMRO on a forward"); + } + _computeMroSequence(); for (size_t i = 0; i < m_mro.size(); i++) { @@ -990,6 +968,10 @@ class HeldClass : public Type { } for (HeldClass* ancestor: m_mro) { + if (ancestor->isForwardDefined()) { + throw std::runtime_error("Makes no sense to be a subclass of a forward"); + } + ancestor->m_implementors.insert(this); } @@ -1008,6 +990,18 @@ class HeldClass : public Type { } setMagicMethodExistConstants(); + + size_t size = mBytesOfInitializationBits; + + for (auto t: m_members) { + m_byte_offsets.push_back(size); + size += t.getType()->bytecount(); + } + + m_is_default_constructible = ( + m_memberFunctions.find("__init__") == m_memberFunctions.end() + ); + m_size = size; } void updateBytesOfInitBits(); @@ -1120,6 +1114,7 @@ class HeldClass : public Type { ClassDispatchTable* dispatchTableAs(HeldClass* interface) { int64_t offset = getMroIndex(interface); if (offset < 0) { + asm("int3"); throw std::runtime_error("Interface is not an ancestor."); } diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index b3f250c36..c329070c0 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -151,8 +151,14 @@ void OneOfType::updateInternalTypePointersConcrete( ) { std::vector newTypes; std::set seenTypes; + std::set everVisited; std::function visit = [&](Type* t) { + if (everVisited.find(t) != everVisited.end()) { + return; + } + everVisited.insert(t); + if (t->getTypeCategory() == catOneOf) { for (auto subt: ((OneOfType*)t)->getTypes()) { visit(subt); diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index a2ea43fa3..4194a983a 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -590,6 +590,22 @@ class Type { } } + template + static void adaptTypeVisitor(const visitor_type& vis, concrete_type*& ioTypePtr) { + Type* t = ioTypePtr; + vis(t); + if (t != (Type*)ioTypePtr) { + if (t->getTypeCategory() != ioTypePtr->getTypeCategory() && !ioTypePtr->isForward()) { + throw std::runtime_error( + "Somehow, visitor changed the type category from " + + ioTypePtr->getTypeCategoryString() + " to " + t->getTypeCategoryString() + ); + } + + ioTypePtr = (concrete_type*)t; + } + } + template static void updateTypeRefFromGroupMap(T*& toUpdate, const std::map& groupMap) { auto it = groupMap.find(toUpdate); diff --git a/typed_python/TypeOrPyobj.cpp b/typed_python/TypeOrPyobj.cpp index d1ad38c12..17ae4aa7a 100644 --- a/typed_python/TypeOrPyobj.cpp +++ b/typed_python/TypeOrPyobj.cpp @@ -22,6 +22,7 @@ TypeOrPyobj::TypeOrPyobj(Type* t) : mPyObj(nullptr) { if (!mType) { + asm("int3"); throw std::runtime_error("Can't construct a TypeOrPyobj with a null Type"); } } diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index dc6a06d35..4b828c371 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -2,7 +2,8 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, - Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value + Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, + Class, Member ) @@ -238,3 +239,108 @@ def test_create_value_type_with_forward(): assert not isForwardDefined(T_resolved) assert identityHash(T_resolved).hex() == identityHash(Value(int)).hex() assert T_resolved is Value(int) + + +def test_create_class_with_forward(): + def makeClass(T): + F = Forward("X") + + class C(Class): + m = Member(F) + + assert isForwardDefined(C) + + F.define(T) + + return resolveForwardDefinedType(C) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_create_concrete_class(): + def makeClass(T): + class C(Class): + m = Member(T) + + assert not isForwardDefined(C) + + return C + + assert makeClass(int) is makeClass(int) + assert makeClass(int) is not makeClass(str) + + +def test_create_class_with_recursion(): + def makeClass(T): + C_fwd = Forward("C") + + class C(Class): + c = Member(OneOf(None, C_fwd)) + m = Member(T) + + assert isForwardDefined(C_fwd) + assert isForwardDefined(C) + + C_fwd.define(C) + + return resolveForwardDefinedType(C) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_create_class_with_nonforward_base(): + def makeClass(T): + C_child_fwd = Forward("C_child") + + class CBase(Class): + m = Member(T) + + class CChild(CBase): + pass + + assert not isForwardDefined(CBase) + assert isForwardDefined(C_child_fwd) + + C_child_fwd.define(CChild) + + return resolveForwardDefinedType(CChild) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +def test_create_class_with_forward_base(): + def makeClass(T): + C_base_fwd = Forward("C_base") + C_child_fwd = Forward("C_child") + + class CBase(Class): + c = Member(OneOf(None, C_child_fwd)) + + class CChild(CBase): + m = Member(T) + + assert isForwardDefined(C_base_fwd) + assert isForwardDefined(C_child_fwd) + + C_child_fwd.define(CChild) + C_base_fwd.define(CBase) + + return resolveForwardDefinedType(CChild) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" From 85f2f973b32b0a4bde20e9146bfbb8ba6140f260 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 1 Jun 2023 22:44:16 +0000 Subject: [PATCH 15/83] Function instances with no state can be 'forward defined' and resolved. --- typed_python/_types.cpp | 15 ++++++ typed_python/type_construction_test.py | 64 ++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index a960a3fc5..125965c20 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2502,6 +2502,11 @@ PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { } PyObjectHolder a1(PyTuple_GetItem(args, 0)); + Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + if (typeOfArg && typeOfArg->isFunction() && typeOfArg->bytecount() == 0) { + return incref(typeOfArg->isForwardDefined() ? Py_True : Py_False); + } + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); if (!t) { @@ -2519,6 +2524,16 @@ PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { } PyObjectHolder a1(PyTuple_GetItem(args, 0)); + Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + + if (typeOfArg && typeOfArg->isFunction() && typeOfArg->bytecount() == 0) { + return ::translateExceptionToPyObject([&]() { + Type* newType = typeOfArg->forwardResolvesTo(); + + return PyInstance::initialize(newType, [&](instance_ptr data) {}); + }); + } + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); if (!t) { diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 4b828c371..8d20ec214 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -227,6 +227,26 @@ def f(x: t): assert makeFforward(int) is makeFforward(int) +def test_instances_of_forward_function_are_forward(): + T = Forward("T") + + @Function + def f(x: T): + return x + + assert isForwardDefined(f) + T.define(int) + + f2 = resolveForwardDefinedType(f) + + assert not isForwardDefined(f2) + + assert f2(1) == 1 + + with pytest.raises(TypeError): + f2("hi") + + def test_create_value_type_with_forward(): F = Forward("X") T = Value(F) @@ -344,3 +364,47 @@ class CChild(CBase): assert makeClass(int)().m == 0 assert makeClass(str)().m == "" + + +@pytest.mark.skip(reason='TODO: make this work') +def test_create_class_with_fully_forward_base(): + def makeClass(T): + C_base_fwd = Forward("C_base") + C_child_fwd = Forward("C_child") + + class CBase(Class): + c = Member(OneOf(None, C_child_fwd)) + + class CChild(C_base_fwd): + m = Member(T) + + assert isForwardDefined(C_base_fwd) + assert isForwardDefined(C_child_fwd) + + C_child_fwd.define(CChild) + C_base_fwd.define(CBase) + + return resolveForwardDefinedType(CChild) + + assert makeClass(int) is not makeClass(float) + assert makeClass(int) is makeClass(int) + + assert makeClass(int)().m == 0 + assert makeClass(str)().m == "" + + +# TODO: +# 1. can we get better names and info for forward-defined types? what does ther +# partially defined state look like +# 2. the CVOV needs to be able to look into TP instances. Value(TupleOf(int)((1, 2, 3))) for +# instance ought to be visible and compilable +# 3. can regular python classes and functions be 'forward declared'. What do we do if we +# forward resolve them? +# 4. serialization +# 5. type functions +# 6. are we accidentally creating and installing forward subclasses in the vtable? +# maybe we need a second pass to wire in 'accepted' forward types +# 7. make sure we thoroughly test the idea that we create a new forward graph with +# part of the graph being new, part being old +# 8. clean-up in the whole Instance/PyInstance layer - kind of nasty +# 9. we should memoize instances of statless Function objects and instances of regular TP lists/tuple of, etc. From dc714e00a3119a9dbf3114f70eb8ecbf20cc3cdb Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 2 Jun 2023 15:18:06 +0000 Subject: [PATCH 16/83] [wip] python subclass? --- typed_python/AlternativeMatcherType.hpp | 2 - typed_python/CompositeType.cpp | 91 ------ typed_python/ConcreteAlternativeType.hpp | 2 - typed_python/PythonSubclassType.cpp | 19 +- typed_python/PythonSubclassType.hpp | 39 ++- typed_python/RefToType.hpp | 6 +- typed_python/Type.hpp | 4 - typed_python/TypedCellType.hpp | 66 ++-- typed_python/_types.cpp | 389 ++++++++++++----------- typed_python/type_construction_test.py | 35 +- 10 files changed, 298 insertions(+), 355 deletions(-) diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index b03d7b7f4..23736873d 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -33,8 +33,6 @@ class AlternativeMatcher : public Type { m_alternative = inAlternative; m_size = inAlternative->bytecount(); m_is_simple = false; - - endOfConstructorInitialization(); // finish initializing the type object. } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { diff --git a/typed_python/CompositeType.cpp b/typed_python/CompositeType.cpp index 88072ae35..a9efba1ce 100644 --- a/typed_python/CompositeType.cpp +++ b/typed_python/CompositeType.cpp @@ -36,43 +36,6 @@ bool CompositeType::isBinaryCompatibleWithConcrete(Type* other) { return true; } -/* -bool CompositeType::_updateAfterForwardTypesChanged() { - bool is_default_constructible = true; - size_t size = 0; - - m_byte_offsets.clear(); - - for (auto t: m_types) { - m_byte_offsets.push_back(size); - size += t->bytecount(); - } - - for (auto t: m_types) { - if (!t->is_default_constructible()) { - is_default_constructible = false; - } - } - - m_serialize_typecodes.clear(); - m_serialize_typecodes_to_position.clear(); - for (int i = 0; i < m_types.size(); i++) { - m_serialize_typecodes.push_back(i); - m_serialize_typecodes_to_position[i] = i; - } - - bool anyChanged = ( - m_is_default_constructible != is_default_constructible || - size != m_size - ); - - m_size = size; - m_is_default_constructible = is_default_constructible; - - return anyChanged; -} -*/ - bool CompositeType::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (pyComparisonOp == Py_NE) { return !cmp(left, right, Py_EQ, suppressExceptions); @@ -192,57 +155,3 @@ std::string Tuple::computeRecursiveNameConcrete(TypeStack& typeStack) { return name; } - - -/* -bool NamedTuple::_updateAfterForwardTypesChanged() { - bool anyChanged = ((CompositeType*)this)->_updateAfterForwardTypesChanged(); - - std::string oldName = m_name; - - if (m_is_recursive_forward) { - m_name = m_recursive_name; - m_stripped_name = ""; - } else { - m_name = "NamedTuple("; - m_stripped_name = ""; - - if (m_types.size() != m_names.size()) { - throw std::logic_error("Names mismatched with types!"); - } - - for (long k = 0; k < m_types.size();k++) { - if (k) { - m_name += ", "; - } - m_name += m_names[k] + "=" + m_types[k]->name(true); - } - m_name += ")"; - } - - return anyChanged || (oldName != m_name); -} - -bool Tuple::_updateAfterForwardTypesChanged() { - bool anyChanged = ((CompositeType*)this)->_updateAfterForwardTypesChanged(); - - std::string oldName = m_name; - - if (m_is_recursive_forward) { - m_name = m_recursive_name; - m_stripped_name = ""; - } else { - m_name = "Tuple("; - for (long k = 0; k < m_types.size();k++) { - if (k) { - m_name += ", "; - } - m_name += m_types[k]->name(true); - } - m_name += ")"; - m_stripped_name = ""; - } - - return anyChanged || m_name != oldName; -} -*/ \ No newline at end of file diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index 6eb64b8f1..2455f635a 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -121,8 +121,6 @@ class ConcreteAlternative : public Type { v.visitTopo(m_alternative); } - bool _updateAfterForwardTypesChanged(); - typed_python_hash_type hash(instance_ptr left) { return m_alternative->hash(left); } diff --git a/typed_python/PythonSubclassType.cpp b/typed_python/PythonSubclassType.cpp index 2929e3467..429865838 100644 --- a/typed_python/PythonSubclassType.cpp +++ b/typed_python/PythonSubclassType.cpp @@ -34,24 +34,13 @@ bool PythonSubclass::isBinaryCompatibleWithConcrete(Type* other) { PythonSubclass* PythonSubclass::Make(Type* base, PyTypeObject* pyType) { PyEnsureGilAcquired getTheGil; - static std::map m; - - auto it = m.find(pyType); - - if (it == m.end()) { - it = m.insert( - std::make_pair(pyType, new PythonSubclass(base, pyType)) - ).first; + if (base->isForwardDefined()) { + return new PythonSubclass(base, pyType); } - if (it->second->getBaseType() != base) { - throw std::runtime_error( - "Expected to find the same base type. Got " - + it->second->getBaseType()->name() + " != " + base->name() - ); - } + PythonSubclass* res = new PythonSubclass(base, pyType); - return it->second; + return (PythonSubclass*)res->forwardResolvesTo(); } diff --git a/typed_python/PythonSubclassType.hpp b/typed_python/PythonSubclassType.hpp index 5e229a056..7f1d3e0da 100644 --- a/typed_python/PythonSubclassType.hpp +++ b/typed_python/PythonSubclassType.hpp @@ -19,10 +19,17 @@ #include "Type.hpp" class PythonSubclass : public Type { + PythonSubclass() : Type(TypeCategory::catPythonSubclass) + { + m_needs_post_init = true; + } + public: PythonSubclass(Type* base, PyTypeObject* typePtr) : Type(TypeCategory::catPythonSubclass) { + m_is_forward_defined = true; + m_base = base; mTypeRep = (PyTypeObject*)incref((PyObject*)typePtr); m_name = typePtr->tp_name; @@ -48,8 +55,6 @@ class PythonSubclass : public Type { if (!PyFunction_Check(m_hashFun)) { m_hashFun = nullptr; } - - endOfConstructorInitialization(); // finish initializing the type object. } template @@ -78,19 +83,29 @@ class PythonSubclass : public Type { visitor(m_base); } - bool _updateAfterForwardTypesChanged() { - size_t size = m_base->bytecount(); - bool is_default_constructible = m_base->is_default_constructible(); + void postInitializeConcrete() { + m_base->postInitialize(); - bool anyChanged = ( - size != m_size || - m_is_default_constructible != is_default_constructible - ); + m_size = m_base->bytecount(); + m_is_default_constructible = m_base->is_default_constructible(); + } + + Type* cloneForForwardResolutionConcrete() { + return new PythonSubclass(); + } - m_size = size; - m_is_default_constructible = is_default_constructible; + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + PythonSubclass* toDup = (PythonSubclass*)forwardDefinitionOfSelf; + + mTypeRep = incref(toDup->mTypeRep); + m_name = toDup->m_name; + m_reprFun = toDup->m_reprFun; + m_hashFun = toDup->m_hashFun; + m_base = toDup->m_base; + } - return anyChanged; + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(m_base, groupMap); } typed_python_hash_type hash(instance_ptr left); diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index aa7b5bf81..e1b4c405d 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -77,12 +77,8 @@ class RefTo : public Type { m_element_type = ((RefTo*)forwardDefinitionOfSelf)->m_element_type; } - void updateInternalTypePointersConcrete(const std::map& groupMap) { - auto it = groupMap.find(m_element_type); - if (it != groupMap.end()) { - m_element_type = it->second; - } + updateTypeRefFromGroupMap(m_element_type, groupMap); } Type* cloneForForwardResolutionConcrete() { diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 4194a983a..c315792e1 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -757,10 +757,6 @@ class Type { }); } - // called after each type has initialized its internals - // TODO: remove this - void endOfConstructorInitialization() {}; - // call subtype.copy_constructor void copy_constructor(instance_ptr self, instance_ptr other); diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index 867af2a14..98c3fa02a 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -21,6 +21,10 @@ //wraps a python Cell class TypedCellType : public Type { + TypedCellType() : Type(TypeCategory::catTypedCell) + { + m_needs_post_init = true; + } public: class layout { public: @@ -35,15 +39,31 @@ class TypedCellType : public Type { Type(TypeCategory::catTypedCell), mHeldType(heldType) { - m_name = std::string("TypedCell(") + heldType->name(true) + ")"; + m_is_forward_defined = true; m_is_simple = false; m_size = sizeof(layout*); m_is_default_constructible = true; + } - endOfConstructorInitialization(); // finish initializing the type object. + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + return "TypedCell(" + mHeldType->computeRecursiveName(typeStack) + ")"; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + mHeldType = ((TypedCellType*)forwardDefinitionOfSelf)->mHeldType; + } + + void updateInternalTypePointersConcrete(const std::map& groupMap) { + updateTypeRefFromGroupMap(mHeldType, groupMap); + } + + void postInitializeConcrete() {} + + Type* cloneForForwardResolutionConcrete() { + return new TypedCellType(); } template @@ -65,25 +85,10 @@ class TypedCellType : public Type { visitor(mHeldType); } - bool _updateAfterForwardTypesChanged() { - std::string newName = std::string("TypedCell(") + mHeldType->name(true) + ")"; - - bool nameChanged = newName != m_name; - - m_name = newName; - m_stripped_name = ""; - - return nameChanged; - } - int64_t refcount(instance_ptr self) const { return getLayoutPtr(self)->refcount; } - void _updateTypeMemosAfterForwardResolution() { - TypedCellType::Make(mHeldType, this); - } - layout_ptr& getLayoutPtr(instance_ptr self) const { return *(layout**)self; } @@ -238,24 +243,25 @@ class TypedCellType : public Type { return getLayoutPtr(self)->data; } - static TypedCellType* Make(Type* t, TypedCellType* knownType = nullptr) { - PyEnsureGilAcquired getTheGil; + static TypedCellType* Make(Type* elt) { + if (elt->isForwardDefined()) { + return new TypedCellType(elt); + } - typedef Type* keytype; + PyEnsureGilAcquired getTheGil; - static std::map m; + static std::map memo; - auto it = m.find(t); - if (it == m.end()) { - it = m.insert( - std::make_pair( - t, - knownType ? knownType : new TypedCellType(t) - ) - ).first; + auto it = memo.find(elt); + if (it != memo.end()) { + return it->second; } - return it->second; + TypedCellType* res = new TypedCellType(elt); + TypedCellType* concrete = (TypedCellType*)res->forwardResolvesTo(); + + memo[elt] = concrete; + return concrete; } Type* getHeldType() const { diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 125965c20..8e85bb881 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -871,237 +871,239 @@ PyObject *MakeAlternativeMatcherType(PyObject* nullValue, PyObject* args) { } PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 6 && PyTuple_Size(args) != 2) { - PyErr_SetString(PyExc_TypeError, "Function takes 2 or 6 arguments"); - return NULL; - } + return translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 6 && PyTuple_Size(args) != 2) { + PyErr_SetString(PyExc_TypeError, "Function takes 2 or 6 arguments"); + throw PythonExceptionSet(); + } - Function* resType; + Function* resType; - if (PyTuple_Size(args) == 2) { - PyObjectHolder a0(PyTuple_GetItem(args, 0)); - PyObjectHolder a1(PyTuple_GetItem(args, 1)); + if (PyTuple_Size(args) == 2) { + PyObjectHolder a0(PyTuple_GetItem(args, 0)); + PyObjectHolder a1(PyTuple_GetItem(args, 1)); - Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); - Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); + Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); + Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); - if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); - return NULL; - } - if (!t1 || t1->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_SetString(PyExc_TypeError, "Expected second argument to be a function"); - return NULL; - } + if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); + throw PythonExceptionSet(); + } + if (!t1 || t1->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected second argument to be a function"); + throw PythonExceptionSet(); + } - resType = Function::merge((Function*)t0, (Function*)t1); - } else { - PyObjectHolder nameObj(PyTuple_GetItem(args, 0)); - if (!PyUnicode_Check(nameObj)) { - PyErr_SetString(PyExc_TypeError, "First arg should be a string."); - return NULL; - } - PyObjectHolder qualnameObj(PyTuple_GetItem(args, 1)); - if (!PyUnicode_Check(qualnameObj)) { - PyErr_SetString(PyExc_TypeError, "First arg should be a string."); - return NULL; - } - PyObjectHolder retType(PyTuple_GetItem(args, 2)); - PyObjectHolder funcObj(PyTuple_GetItem(args, 3)); - PyObjectHolder argTuple(PyTuple_GetItem(args, 4)); + resType = Function::merge((Function*)t0, (Function*)t1); + } else { + PyObjectHolder nameObj(PyTuple_GetItem(args, 0)); + if (!PyUnicode_Check(nameObj)) { + PyErr_SetString(PyExc_TypeError, "First arg should be a string."); + throw PythonExceptionSet(); + } + PyObjectHolder qualnameObj(PyTuple_GetItem(args, 1)); + if (!PyUnicode_Check(qualnameObj)) { + PyErr_SetString(PyExc_TypeError, "First arg should be a string."); + throw PythonExceptionSet(); + } + PyObjectHolder retType(PyTuple_GetItem(args, 2)); + PyObjectHolder funcObj(PyTuple_GetItem(args, 3)); + PyObjectHolder argTuple(PyTuple_GetItem(args, 4)); - int assumeClosureGlobal = PyObject_IsTrue(PyTuple_GetItem(args, 5)); + int assumeClosureGlobal = PyObject_IsTrue(PyTuple_GetItem(args, 5)); - if (assumeClosureGlobal == -1) { - return NULL; - } + if (assumeClosureGlobal == -1) { + throw PythonExceptionSet(); + } - if (!PyFunction_Check(funcObj)) { - PyErr_SetString(PyExc_TypeError, "Third arg should be a function object"); - return NULL; - } + if (!PyFunction_Check(funcObj)) { + PyErr_SetString(PyExc_TypeError, "Third arg should be a function object"); + throw PythonExceptionSet(); + } - Type* rType = 0; - PyObject* rSignature = 0; + Type* rType = 0; + PyObject* rSignature = 0; - if (retType != Py_None) { - rType = PyInstance::unwrapTypeArgToTypePtr(retType); - if (!rType) { - if (!PyCallable_Check(retType)) { - PyErr_SetString(PyExc_TypeError, "Expected second argument to be None, a type, or a callable type signature function"); - return NULL; + if (retType != Py_None) { + rType = PyInstance::unwrapTypeArgToTypePtr(retType); + if (!rType) { + if (!PyCallable_Check(retType)) { + PyErr_SetString(PyExc_TypeError, "Expected second argument to be None, a type, or a callable type signature function"); + throw PythonExceptionSet(); + } + PyErr_Clear(); + rSignature = retType; } - PyErr_Clear(); - rSignature = retType; } - } - - if (!PyTuple_Check(argTuple)) { - PyErr_SetString(PyExc_TypeError, "Expected fourth argument to be a tuple of args"); - return NULL; - } - std::vector argList; - - for (long k = 0; k < PyTuple_Size(argTuple); k++) { - PyObjectHolder kTup(PyTuple_GetItem(argTuple, k)); - if (!PyTuple_Check(kTup) || PyTuple_Size(kTup) != 5) { - PyErr_SetString(PyExc_TypeError, "Argtuple elements should be tuples of five things."); - return NULL; + if (!PyTuple_Check(argTuple)) { + PyErr_SetString(PyExc_TypeError, "Expected fourth argument to be a tuple of args"); + throw PythonExceptionSet(); } - PyObjectHolder k0(PyTuple_GetItem(kTup, 0)); - PyObjectHolder k1(PyTuple_GetItem(kTup, 1)); - PyObjectHolder k2(PyTuple_GetItem(kTup, 2)); - PyObjectHolder k3(PyTuple_GetItem(kTup, 3)); - PyObjectHolder k4(PyTuple_GetItem(kTup, 4)); + std::vector argList; - if (!PyUnicode_Check(k0)) { - PyErr_Format(PyExc_TypeError, "Argument %S has a name which is not a string.", (PyObject*)k0); - return NULL; - } + for (long k = 0; k < PyTuple_Size(argTuple); k++) { + PyObjectHolder kTup(PyTuple_GetItem(argTuple, k)); + if (!PyTuple_Check(kTup) || PyTuple_Size(kTup) != 5) { + PyErr_SetString(PyExc_TypeError, "Argtuple elements should be tuples of five things."); + throw PythonExceptionSet(); + } - Type* argT = nullptr; - if (k1 != Py_None) { - argT = PyInstance::unwrapTypeArgToTypePtr(k1); - if (!argT) { - PyErr_Format(PyExc_TypeError, "Argument %S has a type argument %S which should be None or a Type.", k0->ob_type, k1->ob_type); - return NULL; + PyObjectHolder k0(PyTuple_GetItem(kTup, 0)); + PyObjectHolder k1(PyTuple_GetItem(kTup, 1)); + PyObjectHolder k2(PyTuple_GetItem(kTup, 2)); + PyObjectHolder k3(PyTuple_GetItem(kTup, 3)); + PyObjectHolder k4(PyTuple_GetItem(kTup, 4)); + + if (!PyUnicode_Check(k0)) { + PyErr_Format(PyExc_TypeError, "Argument %S has a name which is not a string.", (PyObject*)k0); + throw PythonExceptionSet(); } - } - if ((k3 != Py_True && k3 != Py_False) || (k4 != Py_True && k4 != Py_False)) { - PyErr_Format(PyExc_TypeError, "Argument %S has a malformed type tuple", (PyObject*)k0); - return NULL; - } + Type* argT = nullptr; + if (k1 != Py_None) { + argT = PyInstance::unwrapTypeArgToTypePtr(k1); + if (!argT) { + PyErr_Format(PyExc_TypeError, "Argument %S has a type argument %S which should be None or a Type.", k0->ob_type, k1->ob_type); + throw PythonExceptionSet(); + } + } - PyObject* val = nullptr; - if (k2 != Py_None) { - if (!PyTuple_Check(k2) || PyTuple_Size(k2) != 1) { + if ((k3 != Py_True && k3 != Py_False) || (k4 != Py_True && k4 != Py_False)) { PyErr_Format(PyExc_TypeError, "Argument %S has a malformed type tuple", (PyObject*)k0); - return NULL; + throw PythonExceptionSet(); } - val = PyTuple_GetItem(k2,0); - } + PyObject* val = nullptr; + if (k2 != Py_None) { + if (!PyTuple_Check(k2) || PyTuple_Size(k2) != 1) { + PyErr_Format(PyExc_TypeError, "Argument %S has a malformed type tuple", (PyObject*)k0); + throw PythonExceptionSet(); + } - if (val) { - incref(val); - } + val = PyTuple_GetItem(k2,0); + } - argList.push_back(Function::FunctionArg( - PyUnicode_AsUTF8(k0), - argT, - val, - k3 == Py_True, - k4 == Py_True - )); - } + if (val) { + incref(val); + } - std::string moduleName = ""; + argList.push_back(Function::FunctionArg( + PyUnicode_AsUTF8(k0), + argT, + val, + k3 == Py_True, + k4 == Py_True + )); + } - if (PyObject_HasAttrString(funcObj, "__module__")) { - PyObject* pyModulename = PyObject_GetAttrString(funcObj, "__module__"); + std::string moduleName = ""; - if (!pyModulename) { - return NULL; - } + if (PyObject_HasAttrString(funcObj, "__module__")) { + PyObject* pyModulename = PyObject_GetAttrString(funcObj, "__module__"); - if (PyUnicode_Check(pyModulename)) { - moduleName = PyUnicode_AsUTF8(pyModulename); - } + if (!pyModulename) { + throw PythonExceptionSet(); + } - decref(pyModulename); - } + if (PyUnicode_Check(pyModulename)) { + moduleName = PyUnicode_AsUTF8(pyModulename); + } - std::vector overloads; + decref(pyModulename); + } - std::vector closureVarnames; - std::vector closureVarTypes; - std::map globalsInCells; - std::map closureBindings; + std::vector overloads; - PyObject* closure = PyFunction_GetClosure(funcObj); + std::vector closureVarnames; + std::vector closureVarTypes; + std::map globalsInCells; + std::map closureBindings; - if (closure) { - PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); + PyObject* closure = PyFunction_GetClosure(funcObj); - if (!coFreevars) { - return NULL; - } + if (closure) { + PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); - if (!PyTuple_Check(coFreevars)) { - PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not a tuple"); - return NULL; - } + if (!coFreevars) { + throw PythonExceptionSet(); + } - if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { - PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars had a different number of elements than the closure"); - return NULL; - } + if (!PyTuple_Check(coFreevars)) { + PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not a tuple"); + throw PythonExceptionSet(); + } - for (long ix = 0; ix < PyTuple_Size(coFreevars); ix++) { - PyObject* varname = PyTuple_GetItem(coFreevars, ix); - if (!PyUnicode_Check(varname)) { - PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not all strings"); - return NULL; + if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { + PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars had a different number of elements than the closure"); + throw PythonExceptionSet(); } - closureVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); - } - if (assumeClosureGlobal) { - for (long ix = 0; ix < PyTuple_Size(closure); ix++) { - PyObject* cell = PyTuple_GetItem(closure, ix); - if (!PyCell_Check(cell)) { - PyErr_Format(PyExc_TypeError, "Function closure needs to all be cells."); - return NULL; + for (long ix = 0; ix < PyTuple_Size(coFreevars); ix++) { + PyObject* varname = PyTuple_GetItem(coFreevars, ix); + if (!PyUnicode_Check(varname)) { + PyErr_Format(PyExc_TypeError, "f.__code__.co_freevars was not all strings"); + throw PythonExceptionSet(); } + closureVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); + } + + if (assumeClosureGlobal) { + for (long ix = 0; ix < PyTuple_Size(closure); ix++) { + PyObject* cell = PyTuple_GetItem(closure, ix); + if (!PyCell_Check(cell)) { + PyErr_Format(PyExc_TypeError, "Function closure needs to all be cells."); + throw PythonExceptionSet(); + } - globalsInCells[closureVarnames[ix]] = incref(cell); + globalsInCells[closureVarnames[ix]] = incref(cell); + } } - } - else { - for (long ix = 0; ix < PyTuple_Size(closure); ix++) { - closureVarTypes.push_back(PyCellType::Make()); - closureBindings[closureVarnames[ix]] = ( - ClosureVariableBinding() + 0 + ix + ClosureVariableBindingStep::AccessCell() - ); + else { + for (long ix = 0; ix < PyTuple_Size(closure); ix++) { + closureVarTypes.push_back(PyCellType::Make()); + closureBindings[closureVarnames[ix]] = ( + ClosureVariableBinding() + 0 + ix + ClosureVariableBindingStep::AccessCell() + ); + } } } - } - overloads.push_back( - Function::Overload( - PyFunction_GetCode(funcObj), - PyFunction_GetGlobals(funcObj), - PyFunction_GetDefaults(funcObj), - ((PyFunctionObject*)(PyObject*)funcObj)->func_annotations, - globalsInCells, - closureVarnames, - closureBindings, - rType, - rSignature, - argList, - nullptr - ) - ); + overloads.push_back( + Function::Overload( + PyFunction_GetCode(funcObj), + PyFunction_GetGlobals(funcObj), + PyFunction_GetDefaults(funcObj), + ((PyFunctionObject*)(PyObject*)funcObj)->func_annotations, + globalsInCells, + closureVarnames, + closureBindings, + rType, + rSignature, + argList, + nullptr + ) + ); - resType = Function::Make( - PyUnicode_AsUTF8(nameObj), - PyUnicode_AsUTF8(qualnameObj), - moduleName, - overloads, - Tuple::Make({ - assumeClosureGlobal ? - NamedTuple::Make({}, {}) : - NamedTuple::Make(closureVarTypes, closureVarnames) - }), - false, - false - ); - } + resType = Function::Make( + PyUnicode_AsUTF8(nameObj), + PyUnicode_AsUTF8(qualnameObj), + moduleName, + overloads, + Tuple::Make({ + assumeClosureGlobal ? + NamedTuple::Make({}, {}) : + NamedTuple::Make(closureVarTypes, closureVarnames) + }), + false, + false + ); + } - return incref((PyObject*)PyInstance::typeObj(resType)); + return incref((PyObject*)PyInstance::typeObj(resType)); + }); } PyObject *MakeClassType(PyObject* nullValue, PyObject* args) { @@ -2502,19 +2504,20 @@ PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { } PyObjectHolder a1(PyTuple_GetItem(args, 0)); - Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); - if (typeOfArg && typeOfArg->isFunction() && typeOfArg->bytecount() == 0) { - return incref(typeOfArg->isForwardDefined() ? Py_True : Py_False); - } + return translateExceptionToPyObject([&]() { + Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + if (typeOfArg && typeOfArg->isFunction() && typeOfArg->bytecount() == 0) { + return incref(typeOfArg->isForwardDefined() ? Py_True : Py_False); + } - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); - if (!t) { - PyErr_SetString(PyExc_TypeError, "first argument to 'isForwardDefined' must be a type object"); - return NULL; - } + if (!t) { + throw std::runtime_error("first argument to 'isForwardDefined' must be a type object"); + } - return incref(t->isForwardDefined() ? Py_True : Py_False); + return incref(t->isForwardDefined() ? Py_True : Py_False); + }); } PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 8d20ec214..b1e579b52 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member + Class, Member, ConstDict, TypedCell ) @@ -154,6 +154,29 @@ def makeP(): assert makeP().__name__ == 'RefTo(^0)' +def test_recursive_typed_cell(): + def makeP(): + F = Forward() + F.define(TypedCell(F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'TypedCell(^0)' + + +def test_subclass_of_named_tuple(): + def makeP(T): + class N(NamedTuple(x=T)): + def f(self): + return self.x + + assert not isForwardDefined(N) + return N + + assert makeP(int) is makeP(int) + assert makeP(int) is not makeP(str) + + def test_recursive_dict(): def makeP(): F = Forward() @@ -164,6 +187,16 @@ def makeP(): assert makeP().__name__ == 'Dict(int, ^0)' +def test_recursive_const_dict(): + def makeP(): + F = Forward() + F.define(ConstDict(int, F)) + return resolveForwardDefinedType(F) + + assert makeP() is makeP() + assert makeP().__name__ == 'ConstDict(int, ^0)' + + def test_recursive_set(): def makeP(): F = Forward() From 1256cc03288749554448b13bb0e259ac4a11a018 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 2 Jun 2023 18:45:08 +0000 Subject: [PATCH 17/83] more --- typed_python/ConcreteAlternativeType.hpp | 1 + typed_python/type_construction_test.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index 2455f635a..0ed399054 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -44,6 +44,7 @@ class ConcreteAlternative : public Type { } void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + m_base = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_base; m_alternative = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_alternative; m_which = ((ConcreteAlternative*)forwardDefinitionOfSelf)->m_which; } diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index b1e579b52..721c804e9 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -221,6 +221,8 @@ def makeA(): assert makeA() is makeA() assert makeA().__name__ == 'F' + assert makeA().A is makeA().A + makeA().B() def test_cannot_call_forward_function_instances(): From 4e99e5afbf54b0181a2a249356140efbbbbbe5db Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 5 Jun 2023 18:55:28 +0000 Subject: [PATCH 18/83] rip out 'python subclass'. we don't need it any more and it was always a bit of a hack --- typed_python/AllTypes.hpp | 1 - typed_python/MutuallyRecursiveTypeGroup.cpp | 1 + typed_python/PyInstance.cpp | 101 ++-------- typed_python/PyInstance.hpp | 7 - typed_python/PyPythonSubclassInstance.hpp | 83 -------- ...onSerializationContext_deserialization.cpp | 14 -- ...thonSerializationContext_serialization.cpp | 3 - typed_python/PythonSubclassType.cpp | 112 ----------- typed_python/PythonSubclassType.hpp | 181 ------------------ typed_python/Type.cpp | 4 - typed_python/Type.hpp | 5 - typed_python/_types.cpp | 1 - typed_python/all.cpp | 1 - typed_python/type_construction_test.py | 13 -- 14 files changed, 19 insertions(+), 508 deletions(-) delete mode 100644 typed_python/PyPythonSubclassInstance.hpp delete mode 100644 typed_python/PythonSubclassType.cpp delete mode 100644 typed_python/PythonSubclassType.hpp diff --git a/typed_python/AllTypes.hpp b/typed_python/AllTypes.hpp index 1549e8044..26dab01fa 100644 --- a/typed_python/AllTypes.hpp +++ b/typed_python/AllTypes.hpp @@ -37,7 +37,6 @@ #include "ValueType.hpp" #include "AlternativeType.hpp" #include "ConcreteAlternativeType.hpp" -#include "PythonSubclassType.hpp" #include "PythonObjectOfTypeType.hpp" #include "FunctionType.hpp" #include "HeldClassType.hpp" diff --git a/typed_python/MutuallyRecursiveTypeGroup.cpp b/typed_python/MutuallyRecursiveTypeGroup.cpp index b4fdf6af9..016533d37 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.cpp +++ b/typed_python/MutuallyRecursiveTypeGroup.cpp @@ -807,6 +807,7 @@ void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root, Visi static thread_local int count = 0; count++; if (count > 1) { + asm("int3"); throw std::runtime_error( "There should be only one group algo running at once. Somehow, " "our reference to " + root.name() + " wasn't captured correctly." diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 91de4d2f6..8d598e0bd 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -40,7 +40,6 @@ #include "PyNoneInstance.hpp" #include "PyRegisterTypeInstance.hpp" #include "PyValueInstance.hpp" -#include "PyPythonSubclassInstance.hpp" #include "PyPythonObjectOfTypeInstance.hpp" #include "PyOneOfInstance.hpp" #include "PySubclassOfInstance.hpp" @@ -382,40 +381,20 @@ PyObject* PyInstance::tp_new(PyTypeObject *subtype, PyObject *args, PyObject *kw eltType->assertForwardsResolvedSufficientlyToInstantiate(); - if (isSubclassOfNativeType(subtype)) { - PyInstance* self = (PyInstance*)subtype->tp_alloc(subtype, 0); - - try { - self->initializeEmpty(); - - self->initialize([&](instance_ptr data) { - constructFromPythonArguments(data, eltType, args, kwds); - }, eltType); - - return (PyObject*)self; - } catch(...) { - subtype->tp_dealloc((PyObject*)self); - throw; - } - - // not reachable - assert(false); - } else { - if (eltType->isClass() && ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__") - != ((Class*)eltType)->getOwnClassMembers().end()) { - return PyObject_Call( - ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__")->second, - args, - kwds - ); - } + if (eltType->isClass() && ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__") + != ((Class*)eltType)->getOwnClassMembers().end()) { + return PyObject_Call( + ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__")->second, + args, + kwds + ); + } - Instance inst(eltType, [&](instance_ptr tgt) { - constructFromPythonArguments(tgt, eltType, args, kwds); - }); + Instance inst(eltType, [&](instance_ptr tgt) { + constructFromPythonArguments(tgt, eltType, args, kwds); + }); - return extractPythonObject(inst.data(), eltType); - } + return extractPythonObject(inst.data(), eltType); }); } @@ -914,25 +893,6 @@ bool PyInstance::isNativeType(PyTypeObject* typeObj) { return typeObj->tp_as_buffer == bufferProcs(); } -/** - * Return true if the given PyTypeObject* is a subclass of a NativeType. - * This will return false when called with a native type -*/ -// static -bool PyInstance::isSubclassOfNativeType(PyTypeObject* typeObj) { - if (isNativeType(typeObj)) { - return false; - } - - while (typeObj) { - if (isNativeType(typeObj)) { - return true; - } - typeObj = typeObj->tp_base; - } - return false; -} - Type* PyInstance::rootNativeType(PyTypeObject* typeObj) { PyTypeObject* baseType = typeObj; @@ -949,10 +909,6 @@ Type* PyInstance::rootNativeType(PyTypeObject* typeObj) { // static Type* PyInstance::extractTypeFrom(PyTypeObject* typeObj, bool includePrimitives) { - if (isSubclassOfNativeType(typeObj)) { - return PythonSubclass::Make(rootNativeType(typeObj), typeObj); - } - if (isNativeType(typeObj)) { return ((NativeTypeWrapper*)typeObj)->mType; } @@ -1163,9 +1119,6 @@ PyTypeObject* PyInstance::typeCategoryBaseType(Type::TypeCategory category) { } PyTypeObject* PyInstance::typeObjInternal(Type* inType) { - if (inType->getTypeCategory() == ::Type::TypeCategory::catPythonSubclass) { - return inType->getTypeRep(); - } if (inType->getTypeCategory() == ::Type::TypeCategory::catInt64) { return &PyLong_Type; } @@ -1575,7 +1528,6 @@ PyObject* PyInstance::tp_str_concrete() { // static bool PyInstance::typeCanBeSubclassed(Type* t) { return ( - t->getTypeCategory() == Type::TypeCategory::catNamedTuple || t->getTypeCategory() == Type::TypeCategory::catClass ); } @@ -1629,7 +1581,6 @@ PyObject* PyInstance::categoryToPyString(Type::TypeCategory cat) { if (cat == Type::TypeCategory::catConstDict) { static PyObject* res = PyUnicode_FromString("ConstDict"); return res; } if (cat == Type::TypeCategory::catAlternative) { static PyObject* res = PyUnicode_FromString("Alternative"); return res; } if (cat == Type::TypeCategory::catConcreteAlternative) { static PyObject* res = PyUnicode_FromString("ConcreteAlternative"); return res; } - if (cat == Type::TypeCategory::catPythonSubclass) { static PyObject* res = PyUnicode_FromString("PythonSubclass"); return res; } if (cat == Type::TypeCategory::catBoundMethod) { static PyObject* res = PyUnicode_FromString("BoundMethod"); return res; } if (cat == Type::TypeCategory::catAlternativeMatcher) { static PyObject* res = PyUnicode_FromString("AlternativeMatcher"); return res; } if (cat == Type::TypeCategory::catClass) { static PyObject* res = PyUnicode_FromString("Class"); return res; } @@ -1812,30 +1763,14 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { return StringType::Make(); } - if (PyInstance::isSubclassOfNativeType(pyType)) { - Type* nativeT = PyInstance::rootNativeType(pyType); - - if (!nativeT) { - PyErr_SetString(PyExc_TypeError, - ("Type " + std::string(pyType->tp_name) + " looked like a native type subclass, but has no base").c_str() - ); - return NULL; - } - - //this is now a permanent object - incref(typearg); + Type* res = PyInstance::extractTypeFrom(pyType); - return PythonSubclass::Make(nativeT, pyType); + if (res) { + // we have a native type -> return it + return res; } else { - Type* res = PyInstance::extractTypeFrom(pyType); - - if (res) { - // we have a native type -> return it - return res; - } else { - // we have a python type -> wrap it - return PythonObjectOfType::Make(pyType); - } + // we have a python type -> wrap it + return PythonObjectOfType::Make(pyType); } } diff --git a/typed_python/PyInstance.hpp b/typed_python/PyInstance.hpp index c3f40e5a0..9eaceb052 100644 --- a/typed_python/PyInstance.hpp +++ b/typed_python/PyInstance.hpp @@ -52,7 +52,6 @@ class PyFunctionInstance; class PyBoundMethodInstance; class PyNoneInstance; class PyValueInstance; -class PyPythonSubclassInstance; class PyPythonObjectOfTypeInstance; class PyOneOfInstance; class PySubclassOfInstance; @@ -166,8 +165,6 @@ class PyInstance { return f(*(PyConcreteAlternativeInstance*)obj); case Type::TypeCategory::catAlternativeMatcher: return f(*(PyAlternativeMatcherInstance*)obj); - case Type::TypeCategory::catPythonSubclass: - return f(*(PyPythonSubclassInstance*)obj); case Type::TypeCategory::catPythonObjectOfType: return f(*(PyPythonObjectOfTypeInstance*)obj); case Type::TypeCategory::catClass: @@ -247,8 +244,6 @@ class PyInstance { return f((PyAlternativeInstance*)nullptr); case Type::TypeCategory::catConcreteAlternative: return f((PyConcreteAlternativeInstance*)nullptr); - case Type::TypeCategory::catPythonSubclass: - return f((PyPythonSubclassInstance*)nullptr); case Type::TypeCategory::catPythonObjectOfType: return f((PyPythonObjectOfTypeInstance*)nullptr); case Type::TypeCategory::catClass: @@ -568,8 +563,6 @@ class PyInstance { static PyMappingMethods* mappingMethods(Type* t); - static bool isSubclassOfNativeType(PyTypeObject* typeObj); - static Type* rootNativeType(PyTypeObject* typeObj); static bool isNativeType(PyTypeObject* typeObj); diff --git a/typed_python/PyPythonSubclassInstance.hpp b/typed_python/PyPythonSubclassInstance.hpp deleted file mode 100644 index b37b46c28..000000000 --- a/typed_python/PyPythonSubclassInstance.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/****************************************************************************** - Copyright 2017-2019 typed_python Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -#pragma once - -#include "PyInstance.hpp" - -class PyPythonSubclassInstance : public PyInstance { -public: - typedef PythonSubclass modeled_type; - - static void copyConstructFromPythonInstanceConcrete(PythonSubclass* eltType, instance_ptr tgt, PyObject* pyRepresentation, ConversionLevel level) { - std::pair typeAndPtr = extractTypeAndPtrFrom(pyRepresentation); - Type* argType = typeAndPtr.first; - instance_ptr argDataPtr = typeAndPtr.second; - - if (argType && Type::typesEquivalent(argType, eltType)) { - eltType->getBaseType()->copy_constructor(tgt, argDataPtr); - return; - } - - if (argType && Type::typesEquivalent(argType, eltType->getBaseType())) { - eltType->getBaseType()->copy_constructor(tgt, argDataPtr); - return; - } - - PyInstance::copyConstructFromPythonInstanceConcrete(eltType, tgt, pyRepresentation, level); - } - - PyObject* tp_getattr_concrete(PyObject* pyAttrName, const char* attrName) { - return specializeForType((PyObject*)this, [&](auto& subtype) { - return subtype.tp_getattr_concrete(pyAttrName, attrName); - }, type()->getBaseType()); - } - - int tp_setattr_concrete(PyObject* pyAttrName, PyObject* attrVal) { - return specializeForTypeReturningInt((PyObject*)this, [&](auto& subtype) { - return subtype.tp_setattr_concrete(pyAttrName, attrVal); - }, type()->getBaseType()); - } - - static bool pyValCouldBeOfTypeConcrete(modeled_type* eltType, PyObject* pyRepresentation, ConversionLevel level) { - Type* argType = extractTypeFrom(pyRepresentation->ob_type); - - if (argType && Type::typesEquivalent(argType, eltType)) { - return true; - } - - if (argType && Type::typesEquivalent(argType, eltType->getBaseType())) { - return true; - } - - return false; - } - - static void constructFromPythonArgumentsConcrete(Type* t, uint8_t* data, PyObject* args, PyObject* kwargs) { - constructFromPythonArguments(data, (Type*)t->getBaseType(), args, kwargs); - } - - Py_ssize_t mp_and_sq_length_concrete() { - return specializeForTypeReturningSizeT((PyObject*)this, [&](auto& subtype) { - return subtype.mp_and_sq_length_concrete(); - }, type()->getBaseType()); - } - - int pyInquiryConcrete(const char* op, const char* opErrRep) { - // op == '__bool__' - return 1; - } -}; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 2d66b8dd6..fca8c7912 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1555,7 +1555,6 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff category == Type::TypeCategory::catRefTo || category == Type::TypeCategory::catSubclassOf || category == Type::TypeCategory::catTuple || - (category == Type::TypeCategory::catPythonSubclass && fieldNumber == 1) || //named tuples alternate between strings for names and type values (category == Type::TypeCategory::catNamedTuple && (fieldNumber % 2 == 0)) || //alternatives encode one type exactly @@ -1566,8 +1565,6 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff else if (category == Type::TypeCategory::catNamedTuple) { assertWireTypesEqual(wireType, WireType::BYTES); names.push_back(b.readStringObject()); - } else if (category == Type::TypeCategory::catPythonSubclass && fieldNumber == 2) { - obj.steal(deserializePythonObject(b, wireType)); } else if (category == Type::TypeCategory::catPythonObjectOfType && fieldNumber == 1) { obj.steal(deserializePythonObject(b, wireType)); } else if (category == Type::TypeCategory::catValue && fieldNumber == 1) { @@ -1790,17 +1787,6 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff } resultType = ::ConstDictType::Make(types[0], types[1]); } - else if (category == Type::TypeCategory::catPythonSubclass) { - if (!obj || !PyType_Check(obj)) { - throw std::runtime_error("Invalid native type: PythonSubclass needs a python type."); - } - - if (types.size() != 1) { - throw std::runtime_error("Invalid native type: PythonSubclass needs a native type."); - } - - resultType = ::PythonSubclass::Make(types[0], (PyTypeObject*)(PyObject*)obj); - } else if (category == Type::TypeCategory::catPythonObjectOfType) { if (!obj || !PyType_Check(obj)) { throw std::runtime_error("Invalid native type: PythonObjectOfType needs a python type."); diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index ee78db335..b8d578b3b 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -879,9 +879,6 @@ void PythonSerializationContext::serializeNativeTypeInner( } else if (nativeType->getTypeCategory() == Type::TypeCategory::catDict) { serializeNativeType(((DictType*)nativeType)->keyType(), b, 1); serializeNativeType(((DictType*)nativeType)->valueType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPythonSubclass) { - serializeNativeType(((PythonSubclass*)nativeType)->baseType(), b, 1); - serializePythonObject((PyObject*)((PythonSubclass*)nativeType)->pyType(), b, 2); } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTupleOf) { serializeNativeType(((TupleOfType*)nativeType)->getEltType(), b, 1); } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPointerTo) { diff --git a/typed_python/PythonSubclassType.cpp b/typed_python/PythonSubclassType.cpp deleted file mode 100644 index 429865838..000000000 --- a/typed_python/PythonSubclassType.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/****************************************************************************** - Copyright 2017-2019 typed_python Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -#include "AllTypes.hpp" - -bool PythonSubclass::isBinaryCompatibleWithConcrete(Type* other) { - Type* nonPyBase = m_base; - while (nonPyBase->getTypeCategory() == TypeCategory::catPythonSubclass) { - nonPyBase = nonPyBase->getBaseType(); - } - - Type* otherNonPyBase = other; - while (otherNonPyBase->getTypeCategory() == TypeCategory::catPythonSubclass) { - otherNonPyBase = otherNonPyBase->getBaseType(); - } - - return nonPyBase->isBinaryCompatibleWith(otherNonPyBase); -} - -// static -PythonSubclass* PythonSubclass::Make(Type* base, PyTypeObject* pyType) { - PyEnsureGilAcquired getTheGil; - - if (base->isForwardDefined()) { - return new PythonSubclass(base, pyType); - } - - PythonSubclass* res = new PythonSubclass(base, pyType); - - return (PythonSubclass*)res->forwardResolvesTo(); -} - - -typed_python_hash_type PythonSubclass::hash(instance_ptr left) { - if (m_hashFun) { - PyEnsureGilAcquired acquireTheGil; - - PyObjectStealer selfAsObject(PyInstance::initialize(this, [&](instance_ptr selfData) { - this->copy_constructor(selfData, left); - })); - - PyObjectStealer result( - PyObject_CallFunctionObjArgs( - m_hashFun, - //the typecast is necessary since this is a varargs function, and so the - //C-style calling convention doesn't know to transform the PyObjectStealer - //into a PyObject* - (PyObject*)selfAsObject, - NULL - ) - ); - - if (!result) { - throw PythonExceptionSet(); - } - - if (!PyLong_Check(result)) { - throw std::runtime_error("Expected " + this->name() + ".__hash__ to return an integer"); - } - - return PyLong_AsLong(result); - } - - return m_base->hash(left); -} - -void PythonSubclass::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { - if (!isStr && m_reprFun) { - PyEnsureGilAcquired acquireTheGil; - - PyObjectStealer selfAsObject(PyInstance::initialize(this, [&](instance_ptr selfData) { - this->copy_constructor(selfData, self); - })); - - PyObjectStealer result( - PyObject_CallFunctionObjArgs( - m_reprFun, - //the typecast is necessary since this is a varargs function, and so the - //C-style calling convention doesn't know to transform the PyObjectStealer - //into a PyObject* - (PyObject*)selfAsObject, - NULL - ) - ); - - if (!result) { - throw PythonExceptionSet(); - } - - if (!PyUnicode_Check(result)) { - throw std::runtime_error("Expected " + this->name() + ".__repr__ to return a string"); - } - - stream << PyUnicode_AsUTF8(result); - return; - } - - m_base->repr(self, stream, isStr); -} diff --git a/typed_python/PythonSubclassType.hpp b/typed_python/PythonSubclassType.hpp deleted file mode 100644 index 7f1d3e0da..000000000 --- a/typed_python/PythonSubclassType.hpp +++ /dev/null @@ -1,181 +0,0 @@ -/****************************************************************************** - Copyright 2017-2019 typed_python Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -#pragma once - -#include "Type.hpp" - -class PythonSubclass : public Type { - PythonSubclass() : Type(TypeCategory::catPythonSubclass) - { - m_needs_post_init = true; - } - -public: - PythonSubclass(Type* base, PyTypeObject* typePtr) : - Type(TypeCategory::catPythonSubclass) - { - m_is_forward_defined = true; - - m_base = base; - mTypeRep = (PyTypeObject*)incref((PyObject*)typePtr); - m_name = typePtr->tp_name; - - m_reprFun = PyObject_GetAttrString((PyObject*)typePtr, "__repr__"); - - if (!m_reprFun) { - PyErr_Clear(); - } - - // make sure we don't get the __repr__ from our own implementation - // in the base class. We want to get the subclass' version. - if (!PyFunction_Check(m_reprFun)) { - m_reprFun = nullptr; - } - - m_hashFun = PyObject_GetAttrString((PyObject*)typePtr, "__hash__"); - - if (!m_hashFun) { - PyErr_Clear(); - } - - if (!PyFunction_Check(m_hashFun)) { - m_hashFun = nullptr; - } - } - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitHash(ShaHash(1, m_typeCategory)); - v.visitName(m_name); - v.visitTopo(m_base); - v.visitTopo((PyObject*)mTypeRep); - if (mTypeRep->tp_dict) { - v.visitHash(ShaHash(1)); - v.visitDict((PyObject*)mTypeRep->tp_dict); - } else { - v.visitHash(ShaHash(0)); - } - } - - bool isBinaryCompatibleWithConcrete(Type* other); - - template - void _visitContainedTypes(const visitor_type& visitor) { - visitor(m_base); - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - visitor(m_base); - } - - void postInitializeConcrete() { - m_base->postInitialize(); - - m_size = m_base->bytecount(); - m_is_default_constructible = m_base->is_default_constructible(); - } - - Type* cloneForForwardResolutionConcrete() { - return new PythonSubclass(); - } - - void initializeFromConcrete(Type* forwardDefinitionOfSelf) { - PythonSubclass* toDup = (PythonSubclass*)forwardDefinitionOfSelf; - - mTypeRep = incref(toDup->mTypeRep); - m_name = toDup->m_name; - m_reprFun = toDup->m_reprFun; - m_hashFun = toDup->m_hashFun; - m_base = toDup->m_base; - } - - void updateInternalTypePointersConcrete(const std::map& groupMap) { - updateTypeRefFromGroupMap(m_base, groupMap); - } - - typed_python_hash_type hash(instance_ptr left); - - template - void serialize(instance_ptr self, buf_t& buffer, size_t fieldNumber) { - m_base->serialize(self, buffer, fieldNumber); - } - - void deepcopyConcrete( - instance_ptr dest, - instance_ptr src, - DeepcopyContext& context - ) { - m_base->deepcopy(dest, src, context); - } - - template - void visitMROConcrete(const visitor_type& v) { - v(this); - m_base->visitMROConcrete(v); - } - - size_t deepBytecountConcrete(instance_ptr instance, std::unordered_set& alreadyVisited, std::set* outSlabs) { - return m_base->deepBytecount(instance, alreadyVisited, outSlabs); - } - - template - void deserialize(instance_ptr self, buf_t& buffer, size_t wireType) { - m_base->deserialize(self, buffer, wireType); - } - - void repr(instance_ptr self, ReprAccumulator& stream, bool isStr); - - bool cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { - return m_base->cmp(left, right, pyComparisonOp, suppressExceptions); - } - - bool isPODConcrete() { - return m_base->isPOD(); - } - - void constructor(instance_ptr self) { - m_base->constructor(self); - } - - void destroy(instance_ptr self) { - m_base->destroy(self); - } - - void copy_constructor(instance_ptr self, instance_ptr other) { - m_base->copy_constructor(self, other); - } - - void assign(instance_ptr self, instance_ptr other) { - m_base->assign(self, other); - } - - static PythonSubclass* Make(Type* base, PyTypeObject* pyType); - - Type* baseType() const { - return m_base; - } - - PyTypeObject* pyType() const { - return mTypeRep; - } - -private: - PyObject* m_reprFun; - - PyObject* m_hashFun; -}; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 1127861b6..428bc2669 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -220,10 +220,6 @@ bool Type::isBinaryCompatibleWith(Type* other) { return true; } - while (other->getTypeCategory() == TypeCategory::catPythonSubclass) { - other = other->getBaseType(); - } - auto it = mIsBinaryCompatible.find(other); if (it != mIsBinaryCompatible.end()) { return it->second != BinaryCompatibilityCategory::Incompatible; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index c315792e1..8f807b420 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -72,7 +72,6 @@ class ConstDictType; class Alternative; class ConcreteAlternative; class AlternativeMatcher; -class PythonSubclass; class PythonObjectOfType; class SubclassOfType; class Class; @@ -130,7 +129,6 @@ class Type { catConstDict = 22, catAlternative = 23, catConcreteAlternative = 24, //concrete Alternative subclass - catPythonSubclass = 25, //subclass of a typed_python type catPythonObjectOfType = 26, //a python object that matches 'isinstance' on a particular type catBoundMethod = 27, catAlternativeMatcher = 37, @@ -488,7 +486,6 @@ class Type { if (category == Type::TypeCategory::catConstDict) { return "ConstDict"; } if (category == Type::TypeCategory::catAlternative) { return "Alternative"; } if (category == Type::TypeCategory::catConcreteAlternative) { return "ConcreteAlternative"; } - if (category == Type::TypeCategory::catPythonSubclass) { return "PythonSubclass"; } if (category == Type::TypeCategory::catBoundMethod) { return "BoundMethod"; } if (category == Type::TypeCategory::catAlternativeMatcher) { return "AlternativeMatcher"; } if (category == Type::TypeCategory::catClass) { return "Class"; } @@ -561,8 +558,6 @@ class Type { return f(*(Alternative*)this); case catConcreteAlternative: return f(*(ConcreteAlternative*)this); - case catPythonSubclass: - return f(*(PythonSubclass*)this); case catPythonObjectOfType: return f(*(PythonObjectOfType*)this); case catClass: diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 8e85bb881..8fa11713c 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -3631,7 +3631,6 @@ PyInit__types(void) PyModule_AddObject(module, "TypedCell", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catTypedCell))); PyModule_AddObject(module, "PyCell", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPyCell))); PyModule_AddObject(module, "PythonObjectOfType", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPythonObjectOfType))); - PyModule_AddObject(module, "PythonSubclass", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catPythonSubclass))); PyModule_AddObject(module, "SubclassOf", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catSubclassOf))); PyModule_AddObject(module, "Forward", (PyObject*)incref(PyInstance::typeCategoryBaseType(Type::TypeCategory::catForward))); diff --git a/typed_python/all.cpp b/typed_python/all.cpp index 70d282c56..6f0f64b3b 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -57,7 +57,6 @@ compile the entire group all at once. #include "PythonSerializationContext.cpp" #include "PythonSerializationContext_serialization.cpp" #include "PythonSerializationContext_deserialization.cpp" -#include "PythonSubclassType.cpp" #include "StringType.cpp" #include "TupleOrListOfType.cpp" #include "Type.cpp" diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 721c804e9..c3dd826b5 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -164,19 +164,6 @@ def makeP(): assert makeP().__name__ == 'TypedCell(^0)' -def test_subclass_of_named_tuple(): - def makeP(T): - class N(NamedTuple(x=T)): - def f(self): - return self.x - - assert not isForwardDefined(N) - return N - - assert makeP(int) is makeP(int) - assert makeP(int) is not makeP(str) - - def test_recursive_dict(): def makeP(): F = Forward() From 9e24df09ab585da335e88ebadb5f569e1afff9f2 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 5 Jun 2023 20:43:38 +0000 Subject: [PATCH 19/83] Fix some bugs - kind of putting this back together. --- typed_python/AlternativeMatcherType.hpp | 3 +- typed_python/AlternativeType.hpp | 4 +++ typed_python/BoundMethodType.hpp | 2 ++ typed_python/CompositeType.hpp | 5 +++ typed_python/ConcreteAlternativeType.hpp | 2 ++ typed_python/ConstDictType.hpp | 3 ++ typed_python/DictType.hpp | 3 ++ typed_python/FunctionType.hpp | 2 ++ typed_python/SetType.hpp | 3 ++ typed_python/TupleOrListOfType.hpp | 4 ++- typed_python/Type.cpp | 5 +++ typed_python/Type.hpp | 9 ++++++ typed_python/__init__.py | 13 ++++---- .../compiler/native_compiler/native_ast.py | 31 ++++++++++++------- typed_python/type_construction_test.py | 4 ++- typed_python/types_test.py | 12 +++++-- 16 files changed, 81 insertions(+), 24 deletions(-) diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index 23736873d..6d1068def 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -37,7 +37,7 @@ class AlternativeMatcher : public Type { std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return "AlternativeMatcher(" - + m_alternative->computeRecursiveName(typeStack) + + qualname_to_name(m_alternative->computeRecursiveName(typeStack)) + ")"; } @@ -58,6 +58,7 @@ class AlternativeMatcher : public Type { } void postInitializeConcrete() { + m_alternative->postInitialize(); m_size = m_alternative->bytecount(); } diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 81d0288ea..77c7fbc76 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -138,6 +138,10 @@ class Alternative : public Type { } void postInitializeConcrete() { + for (auto& subtype_pair: m_subtypes) { + subtype_pair.second->postInitialize(); + } + m_arg_positions.clear(); m_default_construction_type = nullptr; diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index d1514c723..dd8bf8ac6 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -59,6 +59,8 @@ class BoundMethod : public Type { } void postInitializeConcrete() { + m_first_arg->postInitialize(); + m_is_simple = false; m_size = m_first_arg->bytecount(); m_is_default_constructible = false; diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index da96c5205..6fc3866ba 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -237,6 +237,7 @@ class CompositeType : public Type { void initializeFromConcrete(Type* forwardDefinitionOfSelf) { m_types = ((CompositeType*)forwardDefinitionOfSelf)->m_types; m_names = ((CompositeType*)forwardDefinitionOfSelf)->m_names; + m_nameToIndex = ((CompositeType*)forwardDefinitionOfSelf)->m_nameToIndex; } void updateInternalTypePointersConcrete(const std::map& groupMap) { @@ -249,6 +250,10 @@ class CompositeType : public Type { } void postInitializeConcrete() { + for (auto t: m_types) { + t->postInitialize(); + } + bool is_default_constructible = true; size_t size = 0; diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index 0ed399054..f2d58b41a 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -59,6 +59,8 @@ class ConcreteAlternative : public Type { } void postInitializeConcrete() { + m_alternative->postInitialize(); + if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { throw std::runtime_error( "invalid alternative index: " + diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 7749b00f4..14d2b32c9 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -106,6 +106,9 @@ class ConstDictType : public Type { void postInitializeConcrete() { m_size = sizeof(void*); m_is_default_constructible = true; + } + + void finalizeTypeConcrete() { m_bytes_per_key = m_key->bytecount(); m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); } diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index b2be055cc..db9819a30 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -98,6 +98,9 @@ class DictType : public Type { void postInitializeConcrete() { m_size = sizeof(void*); m_is_default_constructible = true; + } + + void finalizeTypeConcrete() { m_bytes_per_key = m_key->bytecount(); m_bytes_per_key_value_pair = m_key->bytecount() + m_value->bytecount(); } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index da2296346..0f498541e 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1566,6 +1566,8 @@ class Function : public Type { } void postInitializeConcrete() { + mClosureType->postInitialize(); + m_size = mClosureType->bytecount(); m_is_default_constructible = mClosureType->is_default_constructible(); } diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index 221801f53..9c041b522 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -73,6 +73,9 @@ class SetType : public Type { void postInitializeConcrete() { m_size = sizeof(void*); m_is_default_constructible = true; + } + + void finalizeTypeConcrete() { m_bytes_per_el = m_key_type->bytecount(); } diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 899fd8f2a..a18cbd128 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -596,7 +596,9 @@ class TupleOrListOfType : public Type { void initializeFromConcrete(Type* forwardDefinitionOfSelf); - void postInitializeConcrete() {}; + void postInitializeConcrete() { + m_size = sizeof(layout*); + }; std::string computeRecursiveNameConcrete(TypeStack& typeStack); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 428bc2669..bd444f8ba 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -387,6 +387,11 @@ void Type::attemptToResolve() { typeAndSource.first->postInitialize(); } + // let each type update any internal caches it might need before it gets instantiated + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->finalizeType(); + } + // now internalize the types by their hash. For each type, we compute a hash // and then look to see if we've seen it before. We build a lookup table from // each existing type to the internalized type, and then do the same process we did diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 8f807b420..24d75f866 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -1017,6 +1017,8 @@ class Type { ); } + void finalizeTypeConcrete() {} + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return m_name; } @@ -1043,6 +1045,13 @@ class Type { }); } + // update any caches in the type after we've walked over it + void finalizeType() { + this->check([&](auto& subtype) { + subtype.finalizeTypeConcrete(); + }); + } + std::string computeRecursiveName(TypeStack& stack) { if (!m_needs_post_init) { return m_name; diff --git a/typed_python/__init__.py b/typed_python/__init__.py index a94559baf..858a567a1 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -37,12 +37,12 @@ checkType ) from typed_python._types import bytecount, refcount -# from typed_python.module import Module -# from typed_python.type_function import TypeFunction -# from typed_python.hash import sha_hash -# from typed_python.SerializationContext import SerializationContext -# from typed_python.type_filter import TypeFilter -# from typed_python.compiler.typeof import TypeOf +from typed_python.module import Module +from typed_python.type_function import TypeFunction +from typed_python.hash import sha_hash +from typed_python.SerializationContext import SerializationContext +from typed_python.type_filter import TypeFilter +from typed_python.compiler.typeof import TypeOf from typed_python._types import ( Forward, TupleOf, ListOf, Tuple, NamedTuple, OneOf, ConstDict, SubclassOf, Alternative, Value, serialize, deserialize, serializeStream, deserializeStream, @@ -74,7 +74,6 @@ EmbeddedMessage = _types.EmbeddedMessage() PyCell = _types.PyCell() -# TODO: put this back when everything works # from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa # from typed_python.generator import Generator # noqa diff --git a/typed_python/compiler/native_compiler/native_ast.py b/typed_python/compiler/native_compiler/native_ast.py index 171ade17f..ea1a54d65 100644 --- a/typed_python/compiler/native_compiler/native_ast.py +++ b/typed_python/compiler/native_compiler/native_ast.py @@ -15,7 +15,9 @@ from typed_python.compiler.native_compiler.global_variable_definition import ( GlobalVariableMetadata ) -from typed_python import Tuple, TupleOf, Alternative, NamedTuple, OneOf, Forward +from typed_python import ( + Tuple, TupleOf, Alternative, NamedTuple, OneOf, Forward, resolveForwardDefinedType +) import textwrap @@ -86,8 +88,8 @@ def typeZeroConstant(self): raise Exception(f"Can't make a zero value from {self}") -Type = Forward("Type") -Type = Type.define(Alternative( +Type = Forward() +Type.define(Alternative( "Type", # the 'Void' type. should only be used in function signatures. Use an empty # struct if you want the 'empty' data structure instead. @@ -108,6 +110,7 @@ def typeZeroConstant(self): else False, zeroConstant=typeZeroConstant )) +Type = resolveForwardDefinedType(Type) def const_truth_value(c): @@ -143,8 +146,8 @@ def const_str(c): assert False, type(c) -Constant = Forward("Constant") -Constant = Constant.define(Alternative( +Constant = Forward() +Constant.define(Alternative( "Constant", Void={}, Float={'val': float, 'bits': int}, @@ -156,6 +159,7 @@ def const_str(c): truth_value=const_truth_value, __str__=const_str )) +Constant = resolveForwardDefinedType(Constant) UnaryOp = Alternative( "UnaryOp", @@ -197,10 +201,6 @@ def const_str(c): ) -# loads and stores - no assignments -Expression = Forward("Expression") -Teardown = Forward("Teardown") - NamedCallTarget = NamedTuple( name=str, arg_types=TupleOf(Type), @@ -245,6 +245,9 @@ def intm_could_throw(intm): assert False, f"Unrecognized expression intermediate {intm}" +Expression = Forward() +Teardown = Forward() + ExpressionIntermediate = Alternative( "ExpressionIntermediate", Effect={"expr": Expression}, @@ -294,7 +297,7 @@ def teardown_str(self): assert False, type(self) -Teardown = Teardown.define(Alternative( +Teardown.define(Alternative( "Teardown", ByTag={'tag': str, 'expr': Expression}, Always={'expr': Expression}, @@ -558,7 +561,7 @@ def expr_could_throw(self): return False -Expression = Expression.define(Alternative( +Expression.define(Alternative( "Expression", Constant={'val': Constant}, Comment={'comment': str, 'expr': Expression}, @@ -660,6 +663,12 @@ def expr_could_throw(self): )) +ExpressionIntermediate = resolveForwardDefinedType(ExpressionIntermediate) +Expression = resolveForwardDefinedType(Expression) +Teardown = resolveForwardDefinedType(Teardown) +CallTarget = resolveForwardDefinedType(CallTarget) + + def ensureExpr(x): if isinstance(x, int): return const_int_expr(x) diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index c3dd826b5..bfc77269a 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -201,7 +201,8 @@ def makeA(): Alternative( "F", A={'f': F}, - B={} + B={}, + C={'x': int} ) ) return resolveForwardDefinedType(F) @@ -210,6 +211,7 @@ def makeA(): assert makeA().__name__ == 'F' assert makeA().A is makeA().A makeA().B() + makeA().C(x=10) def test_cannot_call_forward_function_instances(): diff --git a/typed_python/types_test.py b/typed_python/types_test.py index 721c62b86..bf5f4230c 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -31,18 +31,22 @@ Float32, SubclassOf, TupleOf, ListOf, OneOf, Tuple, NamedTuple, Dict, ConstDict, Alternative, serialize, deserialize, Class, - TypeFilter, Function, Forward, Set, PointerTo, Entrypoint, Final + TypeFilter, Function, Forward, Set, PointerTo, + # Entrypoint, + Final, + resolveForwardDefinedType ) from typed_python.type_promotion import ( computeArithmeticBinaryResultType, floatness, bitness, isSignedInt ) -from typed_python.test_util import currentMemUsageMb +# from typed_python.test_util import currentMemUsageMb AnAlternative = Alternative("AnAlternative", X={'x': int}) -AForwardAlternative = Forward("AForwardAlternative") +AForwardAlternative = Forward() AForwardAlternative.define(Alternative("AForwardAlternative", Y={}, X={'x': AForwardAlternative})) +AForwardAlternative = resolveForwardDefinedType(AForwardAlternative) def typeFor(t): @@ -1130,6 +1134,8 @@ def test_const_dict_iter_str(self): def test_alternative_matcher_type(self): A = Alternative("A", X=dict(x=int)) + assert type(A.X()).__name__ == "X" + assert type(A.X().matches).__qualname__ == "AlternativeMatcher(X)" assert type(A.X().matches).__name__ == "AlternativeMatcher(X)" assert type(A.X().matches).__typed_python_category__ == "AlternativeMatcher" assert type(A.X().matches).Alternative is A From 4fc4e9b6147b7cac095f4636a55bf92c4b5f6c65 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 5 Jun 2023 20:46:42 +0000 Subject: [PATCH 20/83] Rip out 'isBinaryCompatible' since its no longer necessary. --- typed_python/AlternativeType.cpp | 27 ------ typed_python/AlternativeType.hpp | 2 - typed_python/BytesType.cpp | 8 -- typed_python/BytesType.hpp | 2 - typed_python/ClassType.cpp | 10 --- typed_python/ClassType.hpp | 2 - typed_python/CompositeType.cpp | 20 ----- typed_python/CompositeType.hpp | 2 - typed_python/ConcreteAlternativeType.cpp | 15 ---- typed_python/ConcreteAlternativeType.hpp | 2 - typed_python/ConstDictType.cpp | 11 --- typed_python/ConstDictType.hpp | 2 - typed_python/DictType.cpp | 11 --- typed_python/DictType.hpp | 2 - typed_python/HeldClassType.cpp | 21 ----- typed_python/HeldClassType.hpp | 2 - typed_python/NoneType.hpp | 8 -- typed_python/OneOfType.cpp | 20 ----- typed_python/OneOfType.hpp | 2 - typed_python/PyCellType.hpp | 4 - typed_python/PythonObjectOfTypeType.hpp | 4 - typed_python/RegisterTypes.hpp | 8 -- typed_python/StringType.hpp | 8 -- typed_python/SubclassOfType.hpp | 4 - typed_python/TupleOrListOfType.cpp | 10 --- typed_python/TupleOrListOfType.hpp | 2 - typed_python/Type.cpp | 30 ------- typed_python/Type.hpp | 6 -- typed_python/TypedCellType.hpp | 4 - typed_python/ValueType.hpp | 4 - typed_python/_types.cpp | 24 ------ .../type_wrappers/alternative_wrapper.py | 2 +- typed_python/types_test.py | 83 ------------------- 33 files changed, 1 insertion(+), 361 deletions(-) diff --git a/typed_python/AlternativeType.cpp b/typed_python/AlternativeType.cpp index e158a4e6d..225e0d04f 100644 --- a/typed_python/AlternativeType.cpp +++ b/typed_python/AlternativeType.cpp @@ -16,33 +16,6 @@ #include "AllTypes.hpp" -bool Alternative::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() == TypeCategory::catConcreteAlternative) { - other = other->getBaseType(); - } - - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - Alternative* otherO = (Alternative*)other; - - if (m_subtypes.size() != otherO->m_subtypes.size()) { - return false; - } - - for (long k = 0; k < m_subtypes.size(); k++) { - if (m_subtypes[k].first != otherO->m_subtypes[k].first) { - return false; - } - if (!m_subtypes[k].second->isBinaryCompatibleWith(otherO->m_subtypes[k].second)) { - return false; - } - } - - return true; -} - int64_t Alternative::refcount(instance_ptr i) const { if (m_all_alternatives_empty) { return 0; diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 77c7fbc76..84e56387f 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -213,8 +213,6 @@ class Alternative : public Type { return m_hasGetAttributeMagicMethod; } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/BytesType.cpp b/typed_python/BytesType.cpp index 3a032e9b3..6511f50b6 100644 --- a/typed_python/BytesType.cpp +++ b/typed_python/BytesType.cpp @@ -476,14 +476,6 @@ void BytesType::assign(instance_ptr self, instance_ptr other) { destroy((instance_ptr)&old); } -bool BytesType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; -} - void BytesType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { stream << "b" << "'"; long bytes = count(self); diff --git a/typed_python/BytesType.hpp b/typed_python/BytesType.hpp index b39ff3ff5..120c38129 100644 --- a/typed_python/BytesType.hpp +++ b/typed_python/BytesType.hpp @@ -38,8 +38,6 @@ class BytesType : public Type { m_size = sizeof(layout*); } - bool isBinaryCompatibleWithConcrete(Type* other); - void repr(instance_ptr self, ReprAccumulator& stream, bool isStr); template diff --git a/typed_python/ClassType.cpp b/typed_python/ClassType.cpp index a65290d86..e3d996ed1 100644 --- a/typed_python/ClassType.cpp +++ b/typed_python/ClassType.cpp @@ -16,16 +16,6 @@ #include "AllTypes.hpp" -bool Class::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - Class* otherO = (Class*)other; - - return m_heldClass->isBinaryCompatibleWith(otherO->m_heldClass); -} - instance_ptr Class::eltPtr(instance_ptr self, int64_t ix) const { layout& l = *instanceToLayout(self); return m_heldClass->eltPtr(l.data, ix); diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index 4e82933b4..b815ee9ba 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -133,8 +133,6 @@ class Class : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/CompositeType.cpp b/typed_python/CompositeType.cpp index a9efba1ce..5f6144de9 100644 --- a/typed_python/CompositeType.cpp +++ b/typed_python/CompositeType.cpp @@ -16,26 +16,6 @@ #include "AllTypes.hpp" -bool CompositeType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - CompositeType* otherO = (CompositeType*)other; - - if (m_types.size() != otherO->m_types.size()) { - return false; - } - - for (long k = 0; k < m_types.size(); k++) { - if (!m_types[k]->isBinaryCompatibleWith(otherO->m_types[k])) { - return false; - } - } - - return true; -} - bool CompositeType::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool suppressExceptions) { if (pyComparisonOp == Py_NE) { return !cmp(left, right, Py_EQ, suppressExceptions); diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index 6fc3866ba..b481c6213 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -60,8 +60,6 @@ class CompositeType : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { for (auto& typePtr: m_types) { diff --git a/typed_python/ConcreteAlternativeType.cpp b/typed_python/ConcreteAlternativeType.cpp index d92f96554..1900a4681 100644 --- a/typed_python/ConcreteAlternativeType.cpp +++ b/typed_python/ConcreteAlternativeType.cpp @@ -17,21 +17,6 @@ #include "AllTypes.hpp" #include "../typed_python/Format.hpp" -bool ConcreteAlternative::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() == TypeCategory::catConcreteAlternative) { - ConcreteAlternative* otherO = (ConcreteAlternative*)other; - - return otherO->m_alternative->isBinaryCompatibleWith(m_alternative) && - m_which == otherO->m_which; - } - - if (other->getTypeCategory() == TypeCategory::catAlternative) { - return m_alternative->isBinaryCompatibleWith(other); - } - - return false; -} - void ConcreteAlternative::constructor(instance_ptr self) { assertForwardsResolvedSufficientlyToInstantiate(); diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index f2d58b41a..b1ed7d3be 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -101,8 +101,6 @@ class ConcreteAlternative : public Type { return m_alternative->deepBytecount(instance, alreadyVisited, outSlabs); } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { Type* t = m_alternative; diff --git a/typed_python/ConstDictType.cpp b/typed_python/ConstDictType.cpp index c7b5b750f..57ebe4a08 100644 --- a/typed_python/ConstDictType.cpp +++ b/typed_python/ConstDictType.cpp @@ -16,17 +16,6 @@ #include "AllTypes.hpp" -bool ConstDictType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - ConstDictType* otherO = (ConstDictType*)other; - - return m_key->isBinaryCompatibleWith(otherO->m_key) && - m_value->isBinaryCompatibleWith(otherO->m_value); -} - // static ConstDictType* ConstDictType::Make(Type* key, Type* value) { if (key->isForwardDefined() || value->isForwardDefined()) { diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 14d2b32c9..654d0129c 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -72,8 +72,6 @@ class ConstDictType : public Type { v.visitTopo(m_value); } - bool isBinaryCompatibleWithConcrete(Type* other); - std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return "ConstDict(" + m_key->computeRecursiveName(typeStack) diff --git a/typed_python/DictType.cpp b/typed_python/DictType.cpp index 307a0d5a0..e9b056c13 100644 --- a/typed_python/DictType.cpp +++ b/typed_python/DictType.cpp @@ -16,17 +16,6 @@ #include "AllTypes.hpp" -bool DictType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - DictType* otherO = (DictType*)other; - - return m_key->isBinaryCompatibleWith(otherO->m_key) && - m_value->isBinaryCompatibleWith(otherO->m_value); -} - // static DictType* DictType::Make(Type* key, Type* value) { if (key->isForwardDefined() || value->isForwardDefined()) { diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index db9819a30..59ad1bbe3 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -64,8 +64,6 @@ class DictType : public Type { v.visitTopo(m_value); } - bool isBinaryCompatibleWithConcrete(Type* other); - std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return "Dict(" + m_key->computeRecursiveName(typeStack) diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index 4d0834c41..eed27bd7e 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -30,27 +30,6 @@ MemberDefinition::MemberDefinition( mDefaultValueAsPyobj = PyInstance::extractPythonObject(mDefaultValue); } -bool HeldClass::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - HeldClass* otherO = (HeldClass*)other; - - if (m_members.size() != otherO->m_members.size()) { - return false; - } - - for (long k = 0; k < m_members.size(); k++) { - if (m_members[k].getName() != otherO->m_members[k].getName() || - !m_members[k].getType()->isBinaryCompatibleWith(otherO->m_members[k].getType())) { - return false; - } - } - - return true; -} - void HeldClass::updateBytesOfInitBits() { bool anyMembersWithInitializers = false; for (auto& m: m_members) { diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 588b83aba..e761fbd04 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -482,8 +482,6 @@ class HeldClass : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { for (auto& o: m_members) { diff --git a/typed_python/NoneType.hpp b/typed_python/NoneType.hpp index 7ccce6105..78be0443a 100644 --- a/typed_python/NoneType.hpp +++ b/typed_python/NoneType.hpp @@ -27,14 +27,6 @@ class NoneType : public Type { m_is_default_constructible = true; } - bool isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; - } - template void _visitReferencedTypes(const visitor_type& v) {} diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index c329070c0..d23d9559b 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -16,26 +16,6 @@ #include "AllTypes.hpp" -bool OneOfType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != TypeCategory::catOneOf) { - return false; - } - - OneOfType* otherO = (OneOfType*)other; - - if (m_types.size() != otherO->m_types.size()) { - return false; - } - - for (long k = 0; k < m_types.size(); k++) { - if (!m_types[k]->isBinaryCompatibleWith(otherO->m_types[k])) { - return false; - } - } - - return true; -} - void OneOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { m_types[*((uint8_t*)self)]->repr(self+1, stream, isStr); } diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index f64d8bfc1..61065cb64 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -53,8 +53,6 @@ class OneOfType : public Type { } } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { for (auto& typePtr: m_types) { diff --git a/typed_python/PyCellType.hpp b/typed_python/PyCellType.hpp index 6372f3d8e..de6150dde 100644 --- a/typed_python/PyCellType.hpp +++ b/typed_python/PyCellType.hpp @@ -31,10 +31,6 @@ class PyCellType : public PyObjectHandleTypeBase { m_is_simple = false; } - bool isBinaryCompatibleWithConcrete(Type* other) { - return other == this; - } - template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index 476339d33..3a2ab9424 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -145,10 +145,6 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { v.visitTopo((PyObject*)mPyTypePtr); } - bool isBinaryCompatibleWithConcrete(Type* other) { - return other == this; - } - template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/RegisterTypes.hpp b/typed_python/RegisterTypes.hpp index 1b186d7e7..376822ecf 100644 --- a/typed_python/RegisterTypes.hpp +++ b/typed_python/RegisterTypes.hpp @@ -195,14 +195,6 @@ class RegisterType : public Type { m_needs_post_init = false; } - bool isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; - } - bool isPODConcrete() { return true; } diff --git a/typed_python/StringType.hpp b/typed_python/StringType.hpp index 36765840a..3cb918187 100644 --- a/typed_python/StringType.hpp +++ b/typed_python/StringType.hpp @@ -115,14 +115,6 @@ class StringType : public Type { static layout* createFromString(std::string s); - bool isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - return true; - } - template void _visitReferencedTypes(const visitor_type& v) {} diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index e400ba90f..43bf9bcfd 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -64,10 +64,6 @@ class SubclassOfType : public Type { v.visitTopo(m_subclassOf); } - bool isBinaryCompatibleWithConcrete(Type* other) { - return false; - } - template void _visitContainedTypes(const visitor_type& visitor) { visitor(m_subclassOf); diff --git a/typed_python/TupleOrListOfType.cpp b/typed_python/TupleOrListOfType.cpp index 0455a4d57..9993c5a30 100644 --- a/typed_python/TupleOrListOfType.cpp +++ b/typed_python/TupleOrListOfType.cpp @@ -16,16 +16,6 @@ #include "AllTypes.hpp" -bool TupleOrListOfType::isBinaryCompatibleWithConcrete(Type* other) { - if (other->getTypeCategory() != m_typeCategory) { - return false; - } - - TupleOfType* otherO = (TupleOfType*)other; - - return m_element_type->isBinaryCompatibleWith(otherO->m_element_type); -} - void TupleOrListOfType::repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { PushReprState isNew(stream, self); diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index a18cbd128..46acebc95 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -51,8 +51,6 @@ class TupleOrListOfType : public Type { m_is_forward_defined = true; } - bool isBinaryCompatibleWithConcrete(Type* other); - template void _visitContainedTypes(const visitor_type& visitor) { diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index bd444f8ba..5489f6bef 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -211,36 +211,6 @@ void Type::assign(instance_ptr self, instance_ptr other) { this->check([&](auto& subtype) { subtype.assign(self, other); } ); } -bool Type::isBinaryCompatibleWith(Type* other) { - if (other == this) { - return true; - } - - if (isSubclassOf(other)) { - return true; - } - - auto it = mIsBinaryCompatible.find(other); - if (it != mIsBinaryCompatible.end()) { - return it->second != BinaryCompatibilityCategory::Incompatible; - } - - //mark that we are recursing through this datastructure. we don't want to - //loop indefinitely. - mIsBinaryCompatible[other] = BinaryCompatibilityCategory::Checking; - - bool isCompatible = this->check([&](auto& subtype) { - return subtype.isBinaryCompatibleWithConcrete(other); - }); - - mIsBinaryCompatible[other] = isCompatible ? - BinaryCompatibilityCategory::Compatible : - BinaryCompatibilityCategory::Incompatible - ; - - return isCompatible; -} - bool Type::canConvertToTrivially(Type* otherType) { if (typesEquivalent(this, otherType)) { return true; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 24d75f866..4a7c6c167 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -781,12 +781,6 @@ class Type { return m_is_default_constructible; } - bool isBinaryCompatibleWith(Type* other); - - bool isBinaryCompatibleWithConcrete(Type* other) { - return false; - } - bool isSimple() const { return m_is_simple; } diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index 98c3fa02a..769b6cf94 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -72,10 +72,6 @@ class TypedCellType : public Type { v.visitTopo(mHeldType); } - bool isBinaryCompatibleWithConcrete(Type* other) { - return other == this; - } - template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index a0cedd206..d4af8dc22 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -24,10 +24,6 @@ PyDoc_STRVAR(Value_doc, class Value : public Type { public: - bool isBinaryCompatibleWithConcrete(Type* other) { - return this == other; - } - template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 8fa11713c..5db8d458f 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -3121,29 +3121,6 @@ PyObject *referencedTypes(PyObject* nullValue, PyObject* args) { }); } -PyObject *isBinaryCompatible(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 2) { - PyErr_SetString(PyExc_TypeError, "isBinaryCompatible takes 2 positional arguments"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - PyObjectHolder a2(PyTuple_GetItem(args, 1)); - - Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); - Type* t2 = PyInstance::unwrapTypeArgToTypePtr(a2); - - if (!t1) { - PyErr_SetString(PyExc_TypeError, "first argument to 'isBinaryCompatible' must be a type object"); - return NULL; - } - if (!t2) { - PyErr_SetString(PyExc_TypeError, "second argument to 'isBinaryCompatible' must be a type object"); - return NULL; - } - - return incref(t1->isBinaryCompatibleWith(t2) ? Py_True : Py_False); -} - PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs) { int num_args = PyTuple_Size(args); @@ -3515,7 +3492,6 @@ static PyMethodDef module_methods[] = { {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, - {"isBinaryCompatible", (PyCFunction)isBinaryCompatible, METH_VARARGS, NULL}, {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupDeepRepr", (PyCFunction)recursiveTypeGroupDeepRepr, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/typed_python/compiler/type_wrappers/alternative_wrapper.py b/typed_python/compiler/type_wrappers/alternative_wrapper.py index e0836bd6f..29261769e 100644 --- a/typed_python/compiler/type_wrappers/alternative_wrapper.py +++ b/typed_python/compiler/type_wrappers/alternative_wrapper.py @@ -61,7 +61,7 @@ def convert_comparison(self, context, lhs, op, rhs): if py_code < 0: return super().convert_comparison(context, lhs, op, rhs) - if _types.isBinaryCompatible(lhs.expr_type.typeRepresentation, rhs.expr_type.typeRepresentation): + if lhs.expr_type.typeRepresentation is rhs.expr_type.typeRepresentation: lhs = lhs.ensureIsReference() rhs = rhs.ensureIsReference() diff --git a/typed_python/types_test.py b/typed_python/types_test.py index bf5f4230c..8cbac98b2 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -209,58 +209,6 @@ def check_expected_performance(self, elapsed, expected=1.0): .format(expected=expected, elapsed=elapsed) ) - def test_object_binary_compatibility(self): - ibc = _types.isBinaryCompatible - - self.assertTrue(ibc(type(None), type(None))) - self.assertTrue(ibc(Int8, Int8)) - - NT = NamedTuple(a=int, b=int) - - class X(NamedTuple(a=int, b=int)): - pass - - class Y(NamedTuple(a=int, b=int)): - pass - - self.assertTrue(ibc(X, X)) - self.assertTrue(ibc(X, Y)) - self.assertTrue(ibc(X, NT)) - self.assertTrue(ibc(Y, NT)) - self.assertTrue(ibc(NT, Y)) - - self.assertFalse(ibc(OneOf(int, float), OneOf(float, int))) - self.assertTrue(ibc(OneOf(int, X), OneOf(int, Y))) - - def test_binary_compatibility_incompatible_alternatives(self): - ibc = _types.isBinaryCompatible - - A1 = Alternative("A1", X={'a': int}, Y={'b': float}) - A2 = Alternative("A2", X={'a': int}, Y={'b': str}) - - self.assertTrue(ibc(A1, A1.X)) - self.assertTrue(ibc(A1, A1.Y)) - self.assertTrue(ibc(A1.Y, A1.Y)) - self.assertTrue(ibc(A1.Y, A1)) - self.assertTrue(ibc(A1.X, A1)) - self.assertFalse(ibc(A1.X, A1.Y)) - - self.assertFalse(ibc(A1, A2)) - self.assertFalse(ibc(A1.X, A2.X)) - self.assertFalse(ibc(A1.Y, A2.Y)) - - def test_binary_compatibility_compatible_alternatives(self): - ibc = _types.isBinaryCompatible - - A1 = Alternative("A1", X={'a': int}, Y={'b': float}) - A2 = Alternative("A2", X={'a': int}, Y={'b': float}) - - self.assertTrue(ibc(A1.X, A2.X)) - self.assertTrue(ibc(A1.Y, A2.Y)) - - self.assertFalse(ibc(A1.X, A2.Y)) - self.assertFalse(ibc(A1.Y, A2.X)) - def test_name_of_type_as_value_instances(self): T1 = Alternative("T1") @@ -1804,37 +1752,6 @@ def test_const_dict_of_tuple(self): self.assertTrue(k in x) x[k] - def test_conversion_of_binary_compatible(self): - class T1(NamedTuple(a=int)): - pass - - class T2(NamedTuple(a=int)): - pass - - class T1Comp(NamedTuple(d=ConstDict(str, T1))): - pass - - class T2Comp(NamedTuple(d=ConstDict(str, T1))): - pass - - self.assertTrue(_types.isBinaryCompatible(T1Comp, T2Comp)) - self.assertTrue(_types.isBinaryCompatible(T1, T2)) - - def test_binary_compatible_nested(self): - def make(): - class Interior(NamedTuple(a=int)): - pass - - class Exterior(NamedTuple(a=Interior)): - pass - - return Exterior - - E1 = make() - E2 = make() - - self.assertTrue(_types.isBinaryCompatible(E1, E2)) - def test_python_objects_in_tuples(self): class NormalPyClass: pass From 81aa960df0e3edc836695ee7bcf0b3fd2b910bc0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 6 Jun 2023 21:48:50 +0000 Subject: [PATCH 21/83] Make sure we visit into compiler visible constants. May need to flesh this out a little more. --- typed_python/AlternativeType.hpp | 12 +- typed_python/CompilerVisibleObjectVisitor.hpp | 103 ++++++++++++++++-- typed_python/FunctionType.hpp | 1 + typed_python/HeldClassType.cpp | 43 +++++++- typed_python/TupleOrListOfType.hpp | 2 + typed_python/Type.hpp | 9 ++ typed_python/ValueType.hpp | 2 +- typed_python/__init__.py | 38 +++---- .../compiler/python_object_representation.py | 6 +- .../python_typed_function_wrapper.py | 12 +- .../compiler/type_wrappers/range_wrapper.py | 17 ++- typed_python/type_construction_test.py | 10 +- typed_python/types_test.py | 21 +++- 13 files changed, 222 insertions(+), 54 deletions(-) diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 84e56387f..7f84a95b6 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -129,6 +129,7 @@ class Alternative : public Type { m_hasGetAttributeMagicMethod = selfT->m_hasGetAttributeMagicMethod; m_methods = selfT->m_methods; m_subtypes = selfT->m_subtypes; + m_subtypes_concrete = selfT->m_subtypes_concrete; m_default_construction_ix = selfT->m_default_construction_ix; m_default_construction_type = selfT->m_default_construction_type; } @@ -138,10 +139,6 @@ class Alternative : public Type { } void postInitializeConcrete() { - for (auto& subtype_pair: m_subtypes) { - subtype_pair.second->postInitialize(); - } - m_arg_positions.clear(); m_default_construction_type = nullptr; @@ -213,6 +210,8 @@ class Alternative : public Type { return m_hasGetAttributeMagicMethod; } + bool isBinaryCompatibleWithConcrete(Type* other); + template void _visitContainedTypes(const visitor_type& visitor) { } @@ -360,6 +359,11 @@ class Alternative : public Type { typed_python_hash_type hash(instance_ptr left); + std::pair unwrap(instance_ptr self) { + size_t ix = which(self); + return std::make_pair(m_subtypes[ix].second, eltPtr(self)); + } + instance_ptr eltPtr(instance_ptr self) const; int64_t which(instance_ptr self) const; diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index f98d0cb22..6ab220812 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -568,6 +568,8 @@ class CompilerVisibleObjectVisitor { instance_ptr instance, const visitor_type& visitor ) { + visitor.visitTopo(TypeOrPyobj(objType)); + if (objType->isComposite()) { CompositeType* compType = (CompositeType*)objType; for (long k = 0; k < compType->getTypes().size(); k++) { @@ -580,6 +582,63 @@ class CompilerVisibleObjectVisitor { return; } + if (objType->isFunction()) { + // visit the function's closure + Function* funcT = (Function*)objType; + walkInstance( + funcT->getClosureType(), + instance, + visitor + ); + return; + } + + if (objType->isOneOf()) { + OneOfType* o = (OneOfType*)objType; + auto typeAndData = o->unwrap(instance); + walkInstance(typeAndData.first, typeAndData.second, visitor); + return; + } + + if (objType->isAlternative()) { + Alternative* a = (Alternative*)objType; + visitor.visitHash(ShaHash(a->which(instance))); + auto typeAndData = a->unwrap(instance); + walkInstance(typeAndData.first, typeAndData.second, visitor); + return; + } + + if (objType->isConcreteAlternative()) { + ConcreteAlternative* a = (ConcreteAlternative*)objType; + walkInstance(a->getAlternative(), instance, visitor); + return; + } + + if (objType->isTupleOf()) { + TupleOfType* a = (TupleOfType*)objType; + size_t count = a->count(instance); + + visitor.visitHash(ShaHash(count)); + for (long k = 0; k < count; k++) { + walkInstance(a->getEltType(), a->eltPtr(instance, k), visitor); + } + + return; + } + + if (objType->isConstDict()) { + ConstDictType* a = (ConstDictType*)objType; + size_t count = a->count(instance); + + visitor.visitHash(ShaHash(count)); + for (long k = 0; k < count; k++) { + walkInstance(a->keyType(), a->kvPairPtrKey(instance, k), visitor); + walkInstance(a->valueType(), a->kvPairPtrValue(instance, k), visitor); + } + + return; + } + if (objType->isPyCell()) { static PyCellType* pct = PyCellType::Make(); @@ -592,6 +651,32 @@ class CompilerVisibleObjectVisitor { visitor.visitTopo(PyCell_Get(o)); return; } + + if (objType->isBool()) { + visitor.visitHash(ShaHash(*(bool*)instance ? 1 : 0)); + return; + } + + if (objType->isRegister()) { + visitor.visitHash(ShaHash::SHA1((void*)instance, objType->bytecount())); + return; + } + if (objType->isString()) { + visitor.visitHash(ShaHash(((StringType*)objType)->toUtf8String(instance))); + return; + } + if (objType->isBytes()) { + size_t ct = ((BytesType*)objType)->count(instance); + visitor.visitHash(ShaHash(ct)); + + if (ct) { + visitor.visitHash(ShaHash::SHA1( + ((BytesType*)objType)->eltPtr(instance, 0), + ct + )); + } + return; + } } @@ -642,6 +727,7 @@ class CompilerVisibleObjectVisitor { // don't visit into constants if (isSimpleConstant(obj.pyobj())) { + //TODO: this just looks wrong return; } @@ -650,18 +736,11 @@ class CompilerVisibleObjectVisitor { visitor.visitHash(ShaHash(2)); visitor.visitTopo(argType); - if (argType->isFunction()) { - // visit the function's closure - Function* funcT = (Function*)argType; - instance_ptr dataPtr = ((PyInstance*)obj.pyobj())->dataPtr(); - - walkInstance( - funcT->getClosureType(), - dataPtr, - visitor - ); - } - + walkInstance( + argType, + ((PyInstance*)obj.pyobj())->dataPtr(), + visitor + ); return; } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 0f498541e..3c9e9361d 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1543,6 +1543,7 @@ class Function : public Type { m_is_forward_defined = true; m_is_simple = false; m_doc = Function_doc; + m_name = mRootName; } std::string moduleNameConcrete() { diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index eed27bd7e..8e4ddcd9f 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -478,9 +478,46 @@ HeldClass* HeldClass::Make( // if we refer to forwards. bool anyForward = false; - result->_visitReferencedTypes( - [&](Type* t) { if (t->isForwardDefined() && t != clsType) { anyForward = true; } } - ); + for (auto h: bases) { + if (h->isForwardDefined()) { + anyForward = true; + } + } + for (auto& m: members) { + if (m.getType() && m.getType()->isForwardDefined()) { + // TODO: could ehte member _instance_ be forward? + anyForward = true; + } + } + + for (auto& nameAndT: memberFunctions) { + if (nameAndT.second->isForwardDefined()) { + anyForward = true; + } + } + for (auto& nameAndT: staticFunctions) { + if (nameAndT.second->isForwardDefined()) { + anyForward = true; + } + } + for (auto& nameAndT: propertyFunctions) { + if (nameAndT.second->isForwardDefined()) { + anyForward = true; + } + } + + // TODO: a classMember could contain a forward reference! + // for (auto& nameAndT: classMembers) { + // if (nameAndT.second->isForwardDefined()) { + // anyForward = true; + // } + // } + + for (auto& nameAndT: classMethods) { + if (nameAndT.second->isForwardDefined()) { + anyForward = true; + } + } if (!anyForward) { return (HeldClass*)result->forwardResolvesTo(); diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 46acebc95..a2cf76e56 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -28,6 +28,7 @@ class TupleOrListOfType : public Type { m_is_tuple(isTuple) { m_needs_post_init = true; + m_is_default_constructible = true; } public: @@ -49,6 +50,7 @@ class TupleOrListOfType : public Type { m_is_tuple(isTuple) { m_is_forward_defined = true; + m_is_default_constructible = true; } template diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 4a7c6c167..73ce89655 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -271,6 +271,15 @@ class Type { ); } + bool isUInt8() const { return m_typeCategory == catUInt8; } + bool isUInt16() const { return m_typeCategory == catUInt16; } + bool isUInt32() const { return m_typeCategory == catUInt32; } + bool isUInt64() const { return m_typeCategory == catUInt64; } + bool isInt8() const { return m_typeCategory == catInt8; } + bool isInt16() const { return m_typeCategory == catInt16; } + bool isInt32() const { return m_typeCategory == catInt32; } + bool isInt() const { return m_typeCategory == catInt64; } + bool isRecursive() { return getRecursiveTypeGroupMembers().size() != 1; } diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index d4af8dc22..a9af8545f 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -20,7 +20,7 @@ PyDoc_STRVAR(Value_doc, "Value(x) -> type representing the single immutable value x" - ); +); class Value : public Type { public: diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 858a567a1..31f99a60e 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -74,27 +74,27 @@ EmbeddedMessage = _types.EmbeddedMessage() PyCell = _types.PyCell() -# from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa +from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa -# from typed_python.generator import Generator # noqa +from typed_python.generator import Generator # noqa -# # this has to come at the end to break import cyclic -# from typed_python.lib.map import map # noqa -# from typed_python.lib.pmap import pmap # noqa -# from typed_python.lib.reduce import reduce # noqa +# this has to come at the end to break import cyclic +from typed_python.lib.map import map # noqa +from typed_python.lib.pmap import pmap # noqa +from typed_python.lib.reduce import reduce # noqa -# _types.initializeGlobalStatics() +_types.initializeGlobalStatics() -# # start a background thread to release the GIL for us. Instead of immediately releasing the GIL, -# # we prefer to release it a short time after our C code no longer needs it, in case it -# # reacquires it immediately, which is very slow. -# # this is needed primarily because we often can't tell whether we're about to enter code -# # that acquires and releases the GIL very frequently (say, in a tight loop), which has -# # a huge performance penalty (a few thousand context switches per second!) -# # instead, we have a thread that checks in the background whether any thread wants us -# # to release, and if so, swap it out. This can yield a 10-50x performance improvement -# # when we're acquiring and releasing frequently. -# gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) -# gilReleaseThreadLoop.start() +# start a background thread to release the GIL for us. Instead of immediately releasing the GIL, +# we prefer to release it a short time after our C code no longer needs it, in case it +# reacquires it immediately, which is very slow. +# this is needed primarily because we often can't tell whether we're about to enter code +# that acquires and releases the GIL very frequently (say, in a tight loop), which has +# a huge performance penalty (a few thousand context switches per second!) +# instead, we have a thread that checks in the background whether any thread wants us +# to release, and if so, swap it out. This can yield a 10-50x performance improvement +# when we're acquiring and releasing frequently. +gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) +gilReleaseThreadLoop.start() -# _types.setGilReleaseThreadLoopSleepMicroseconds(500) +_types.setGilReleaseThreadLoopSleepMicroseconds(500) diff --git a/typed_python/compiler/python_object_representation.py b/typed_python/compiler/python_object_representation.py index 3143b7bae..adb202c8b 100644 --- a/typed_python/compiler/python_object_representation.py +++ b/typed_python/compiler/python_object_representation.py @@ -77,7 +77,7 @@ from typed_python.compiler.type_wrappers.repr_wrapper import ReprWrapper from types import ModuleType from typed_python._types import ( - TypeFor, bytecount, prepareArgumentToBePassedToCompiler, allForwardTypesResolved, + TypeFor, bytecount, prepareArgumentToBePassedToCompiler, serialize, deserialize ) from typed_python import ( @@ -152,8 +152,8 @@ def _typedPythonTypeToTypeWrapper(t): assert isinstance(t, type), t - if not allForwardTypesResolved(t): - return UnresolvedForwardTypeWrapper(t) + if isinstance(t, Forward): + raise Exception("Can't compile against Forward types") if not hasattr(t, '__typed_python_category__'): # this is will be a PythonObjectOfType, but we never actually return such an object diff --git a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py index b71cac261..dc709349b 100644 --- a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py +++ b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py @@ -13,9 +13,12 @@ # limitations under the License. -from typed_python import PointerTo, bytecount, NamedTuple, Class, OneOf, ListOf, TupleOf, Tuple, Set, ConstDict, Dict, Value +from typed_python import ( + PointerTo, bytecount, NamedTuple, Class, OneOf, ListOf, TupleOf, Tuple, Set, + ConstDict, Dict, Value, isForwardDefined +) from typed_python.compiler.merge_type_wrappers import mergeTypeWrappers -from typed_python._types import is_default_constructible, allForwardTypesResolved +from typed_python._types import is_default_constructible from typed_python.internals import CellAccess from typed_python.compiler.conversion_level import ConversionLevel from typed_python.compiler.type_wrappers.wrapper import Wrapper @@ -118,8 +121,9 @@ def computeFunctionOverloadReturnType(overload, argTypes, kwargTypes): Returns: NoReturnTypeSpecified, a type, CannotBeDetermined, or SomeInvalidClassReturnType """ - if not allForwardTypesResolved(overload.functionTypeObject): - return NoReturnTypeSpecified + # TODO: what should we be doing here? + if isForwardDefined(overload.functionTypeObject): + raise Exception("Can't compile against forward defined types.") return typeWrapper(overload.functionTypeObject).signatureCalculator.returnTypeForOverload( overload.index, argTypes, kwargTypes diff --git a/typed_python/compiler/type_wrappers/range_wrapper.py b/typed_python/compiler/type_wrappers/range_wrapper.py index efbe758b4..7bc862f5f 100644 --- a/typed_python/compiler/type_wrappers/range_wrapper.py +++ b/typed_python/compiler/type_wrappers/range_wrapper.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typed_python import NamedTuple, Class, Final, PointerTo -from typed_python import pointerTo +from typed_python import Class, Final, PointerTo, Held, Member, pointerTo class Range(Class, Final, __name__='range'): @@ -36,7 +35,12 @@ def __call__(self, start, stop, step): # noqa range = Range() -class RangeCls(NamedTuple(start=int, stop=int, step=int)): +@Held +class RangeCls(Class, Final): + start = Member(int, nonempty=True) + stop = Member(int, nonempty=True) + step = Member(int, nonempty=True) + def __str__(self): return repr(self) @@ -62,7 +66,12 @@ def __typed_python_int_iter_value__(self, x): return self.start + self.step * x -class RangeIterator(NamedTuple(start=int, stop=int, step=int)): +@Held +class RangeIterator(Class, Final): + start = Member(int, nonempty=True) + stop = Member(int, nonempty=True) + step = Member(int, nonempty=True) + def __iter__(self) -> int: resPtr = self.__fastnext__() if resPtr: diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index bfc77269a..a7a7120b6 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell + Class, Member, ConstDict, TypedCell, Final ) @@ -285,6 +285,14 @@ def test_create_value_type_with_forward(): assert T_resolved is Value(int) +def test_create_class_with_methods(): + class C(Class): + def f(self): + return "hi" + + assert not isForwardDefined(C) + + def test_create_class_with_forward(): def makeClass(T): F = Forward("X") diff --git a/typed_python/types_test.py b/typed_python/types_test.py index 8cbac98b2..11537fb44 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -32,14 +32,14 @@ TupleOf, ListOf, OneOf, Tuple, NamedTuple, Dict, ConstDict, Alternative, serialize, deserialize, Class, TypeFilter, Function, Forward, Set, PointerTo, - # Entrypoint, + Entrypoint, Final, resolveForwardDefinedType ) from typed_python.type_promotion import ( computeArithmeticBinaryResultType, floatness, bitness, isSignedInt ) -# from typed_python.test_util import currentMemUsageMb +from typed_python.test_util import currentMemUsageMb AnAlternative = Alternative("AnAlternative", X={'x': int}) @@ -605,9 +605,24 @@ def test_tuple_of_one_of_multi(self): self.assertEqual(tuple(typedThings), someThings) + def test_alternative_as_value(self): + A = Alternative( + "A", + a0=dict(x_0=str, x_1=TupleOf(ConstDict(int, int))), + ) + + V1 = Value(A.a0(x_0='hi')) + V2 = Value(A.a0(x_0='hi2')) + V3 = Value(A.a0(x_0='hi', x_1=[{1: 2}])) + V4 = Value(A.a0(x_0='hi', x_1=[{1: 3}])) + + assert V1 is not V2 + assert V1 is not V3 + assert V3 is not V4 + def test_compound_oneof(self): producer = RandomValueProducer() - producer.addEvenly(1000, 2) + producer.addEvenly(20, 2) for _ in range(1000): vals = (producer.pickRandomly(), producer.pickRandomly(), producer.pickRandomly()) From 1ee7536e35c44160862157937747c9a94ee6424a Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 7 Jun 2023 18:21:23 +0000 Subject: [PATCH 22/83] Fix a few more bugs. types_test passes now. Rip out 'isSimple' and some other cruft from 'Type'. --- typed_python/AlternativeMatcherType.hpp | 12 +- typed_python/AlternativeType.hpp | 8 +- typed_python/BoundMethodType.hpp | 14 +- typed_python/ClassType.hpp | 10 +- typed_python/CompositeType.hpp | 16 +- typed_python/ConcreteAlternativeType.hpp | 8 +- typed_python/ConstDictType.hpp | 7 +- typed_python/DictType.hpp | 7 +- typed_python/EmbeddedMessageType.hpp | 16 +- typed_python/ForwardType.hpp | 5 +- typed_python/FunctionType.hpp | 11 +- typed_python/HeldClassType.hpp | 45 ++- typed_python/OneOfType.cpp | 4 - typed_python/OneOfType.hpp | 7 +- typed_python/PointerToType.hpp | 11 +- typed_python/PyCellType.hpp | 10 +- typed_python/PythonObjectOfTypeType.hpp | 2 - typed_python/RefToType.hpp | 3 +- typed_python/RegisterTypes.hpp | 59 +++- typed_python/SetType.hpp | 7 +- typed_python/SubclassOfType.hpp | 7 +- typed_python/TupleOrListOfType.hpp | 12 +- typed_python/Type.cpp | 22 +- typed_python/Type.hpp | 45 +-- typed_python/TypedCellType.hpp | 3 - typed_python/ValueType.cpp | 3 - typed_python/ValueType.hpp | 4 + typed_python/_types.cpp | 18 -- .../compiler/python_object_representation.py | 5 +- .../unresolved_forward_type_wrapper.py | 131 -------- typed_python/python_ast.py | 290 ++++++++++-------- typed_python/types_test.py | 76 ----- 32 files changed, 384 insertions(+), 494 deletions(-) delete mode 100644 typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index 6d1068def..d83b2db53 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -19,11 +19,15 @@ #include "Type.hpp" #include "ReprAccumulator.hpp" + +PyDoc_STRVAR(AlternativeMatcher_doc, + "AlternativeMatcher: an instance of 'A.matches' where 'A' is an alternative.\n" +); + // Models an alternatives '.matches' result class AlternativeMatcher : public Type { AlternativeMatcher() : Type(TypeCategory::catAlternativeMatcher) { - m_needs_post_init = true; } public: AlternativeMatcher(Type* inAlternative) : Type(TypeCategory::catAlternativeMatcher) @@ -32,7 +36,10 @@ class AlternativeMatcher : public Type { m_is_default_constructible = false; m_alternative = inAlternative; m_size = inAlternative->bytecount(); - m_is_simple = false; + } + + const char* docConcrete() { + return AlternativeMatcher_doc; } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { @@ -58,7 +65,6 @@ class AlternativeMatcher : public Type { } void postInitializeConcrete() { - m_alternative->postInitialize(); m_size = m_alternative->bytecount(); } diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 7f84a95b6..9ad16b582 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -79,8 +79,6 @@ PyDoc_STRVAR(Alternative_doc, class Alternative : public Type { Alternative() : Type(TypeCategory::catAlternative) { - m_doc = Alternative_doc; - m_needs_post_init = true; } public: @@ -110,14 +108,15 @@ class Alternative : public Type { m_name = name; m_moduleName = moduleName; - m_is_simple = false; m_hasGetAttributeMagicMethod = m_methods.find("__getattribute__") != m_methods.end(); if (m_subtypes.size() > 255) { throw std::runtime_error("Can't have an alternative with more than 255 subelements"); } + } - m_doc = Alternative_doc; + const char* docConcrete() { + return Alternative_doc; } void initializeFromConcrete(Type* forwardDefinitionOfSelf) { @@ -125,7 +124,6 @@ class Alternative : public Type { m_name = selfT->m_name; m_moduleName = selfT->m_moduleName; - m_is_simple = selfT->m_is_simple; m_hasGetAttributeMagicMethod = selfT->m_hasGetAttributeMagicMethod; m_methods = selfT->m_methods; m_subtypes = selfT->m_subtypes; diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index dd8bf8ac6..5e64d311b 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -19,10 +19,15 @@ #include "Type.hpp" #include "ReprAccumulator.hpp" +PyDoc_STRVAR(BoundMethod_doc, + "BoundMethod(T, name)\n\n" + "Holds an instance of 'T' as 't' and has a call method that dispatches to t.name(...)\n" +); + + class BoundMethod : public Type { BoundMethod() : Type(TypeCategory::catBoundMethod) { - m_needs_post_init = true; } public: BoundMethod(Type* inFirstArg, std::string funcName) : Type(TypeCategory::catBoundMethod) @@ -34,6 +39,10 @@ class BoundMethod : public Type { m_size = inFirstArg->bytecount(); } + const char* docConcrete() { + return BoundMethod_doc; + } + template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -59,9 +68,6 @@ class BoundMethod : public Type { } void postInitializeConcrete() { - m_first_arg->postInitialize(); - - m_is_simple = false; m_size = m_first_arg->bytecount(); m_is_default_constructible = false; } diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index b815ee9ba..e5dd715af 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -32,14 +32,12 @@ PyDoc_STRVAR(Class_doc, "\n" "Methods become TypedFunction instances, and support overloading.\n" "Define data members with Member.\n" - ); +); class Class : public Type { // this is the non-forward type resolution version of this Class() : Type(catClass) { - m_needs_post_init = true; - m_doc = Class_doc; } public: @@ -61,8 +59,10 @@ class Class : public Type { m_size = sizeof(layout*); m_is_default_constructible = inClass->is_default_constructible(); m_name = name; - m_doc = Class_doc; - m_is_simple = false; + } + + const char* docConcrete() { + return Class_doc; } void postInitializeConcrete() { diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index b481c6213..9702aae51 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -26,7 +26,6 @@ class CompositeType : public Type { CompositeType(TypeCategory in_typeCategory) : Type(in_typeCategory) { - m_needs_post_init = true; } // construct a forward-defined CompositeType @@ -248,10 +247,6 @@ class CompositeType : public Type { } void postInitializeConcrete() { - for (auto t: m_types) { - t->postInitialize(); - } - bool is_default_constructible = true; size_t size = 0; @@ -332,15 +327,16 @@ class NamedTuple : public CompositeType { // construct a non-forward NamedTuple NamedTuple() : CompositeType(TypeCategory::catNamedTuple) { - m_doc = NamedTuple_doc; } NamedTuple(const std::vector& types, const std::vector& names) : CompositeType(TypeCategory::catNamedTuple, types, names) { assert(types.size() == names.size()); + } - m_doc = NamedTuple_doc; + const char* docConcrete() { + return NamedTuple_doc; } static NamedTuple* Make(const std::vector& types, const std::vector& names) { @@ -369,13 +365,15 @@ class Tuple : public CompositeType { // construct a non-forward Tuple Tuple() : CompositeType(TypeCategory::catTuple) { - } Tuple(const std::vector& types, const std::vector& names) : CompositeType(TypeCategory::catTuple, types, names) { - m_doc = Tuple_doc; + } + + const char* docConcrete() { + return Tuple_doc; } static Tuple* Make(const std::vector& types) { diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index b1ed7d3be..96192593d 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -25,7 +25,6 @@ class ConcreteAlternative : public Type { m_which(0), m_alternative(nullptr) { - m_needs_post_init = true; } public: @@ -37,6 +36,11 @@ class ConcreteAlternative : public Type { m_which(which) { m_is_forward_defined = true; + m_name = m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; + } + + const char* docConcrete() { + return m_alternative->doc(); } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { @@ -59,8 +63,6 @@ class ConcreteAlternative : public Type { } void postInitializeConcrete() { - m_alternative->postInitialize(); - if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { throw std::runtime_error( "invalid alternative index: " + diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 654d0129c..252c39675 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -29,8 +29,6 @@ PyDoc_STRVAR(ConstDictType_doc, class ConstDictType : public Type { ConstDictType() : Type(TypeCategory::catConstDict) { - m_doc = ConstDictType_doc; - m_needs_post_init = true; } public: @@ -51,10 +49,13 @@ class ConstDictType : public Type { m_key(key), m_value(value) { - m_doc = ConstDictType_doc; m_is_forward_defined = true; } + const char* docConcrete() { + return ConstDictType_doc; + } + template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index 59ad1bbe3..95605f7bb 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -33,8 +33,6 @@ class DictType : public Type { // construct a non-forward uninitialized dict DictType() : Type(TypeCategory::catDict) { - m_doc = DictType_doc; - m_needs_post_init = true; } public: // declare a forward-defined Dict @@ -43,10 +41,13 @@ class DictType : public Type { m_key(key), m_value(value) { - m_doc = DictType_doc; m_is_forward_defined = true; } + const char* docConcrete() { + return DictType_doc; + } + template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/EmbeddedMessageType.hpp b/typed_python/EmbeddedMessageType.hpp index 1d283f893..6aec9c629 100644 --- a/typed_python/EmbeddedMessageType.hpp +++ b/typed_python/EmbeddedMessageType.hpp @@ -20,6 +20,17 @@ #include "NullSerializationContext.hpp" #include "SerializationBuffer.hpp" +PyDoc_STRVAR(EmbeddedMessage_doc, + "EmbeddedMessage: represents an embedded message in a serialization graph.\n\n" + "If you have a type T in a type graph and you replace it with EmbeddedMessage and \n" + "deserialize an instance of that type, instead of the actual instance you'll get a\n" + "block of bytes representing the embedded type. This can be used in some circumstances\n" + "to build tooling that can handle messages without knowing their full type. It cannot\n" + "be used in contexts where there may be embedded memos within the subgraph, so use with\n" + "care. This will probably get deprecated at some point.\n" +); + + class EmbeddedMessageType : public BytesType { public: EmbeddedMessageType() @@ -30,6 +41,10 @@ class EmbeddedMessageType : public BytesType { m_typeCategory = TypeCategory::catEmbeddedMessage; } + const char* docConcrete() { + return EmbeddedMessage_doc; + } + static EmbeddedMessageType* Make() { static EmbeddedMessageType* res = new EmbeddedMessageType(); return res; @@ -64,4 +79,3 @@ class EmbeddedMessageType : public BytesType { void postInitializeConcrete() {} }; - diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index cd63ec93f..34b8181a2 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -34,10 +34,13 @@ class Forward : public Type { mTarget(nullptr) { m_name = name; - m_doc = Forward_doc; m_is_forward_defined = true; } + const char* docConcrete() { + return Forward_doc; + } + std::string moduleNameConcrete() { if (mTarget) { return mTarget->moduleNameConcrete(); diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 3c9e9361d..e1be36a02 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1518,9 +1518,10 @@ class Function : public Type { Function() : Type(catFunction) { - m_needs_post_init = true; - m_is_simple = false; - m_doc = Function_doc; + } + + const char* docConcrete() { + return Function_doc; } Function(std::string inName, @@ -1541,8 +1542,6 @@ class Function : public Type { mClosureType(closureType) { m_is_forward_defined = true; - m_is_simple = false; - m_doc = Function_doc; m_name = mRootName; } @@ -1567,8 +1566,6 @@ class Function : public Type { } void postInitializeConcrete() { - mClosureType->postInitialize(); - m_size = mClosureType->bytecount(); m_is_default_constructible = mClosureType->is_default_constructible(); } diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index e761fbd04..22da68b42 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -337,6 +337,19 @@ class VTable { Type* mClassType; }; + +PyDoc_STRVAR(HeldClass_doc, + "HeldClass: subclass Class and decorate with @Held to produce 'held' TP classes.\n" + "\n" + "These behave like TP classes except that their storage is owned not by the heap but\n" + "by the container of the reference. This means that they are faster (because they don't\n" + "have a refcount) but have different semantics. In particular if H is a held class and\n" + "you write\n\n" + " h1 = H()\n" + " h2 = h1\n" + "\nThen h1 and h2 will refer to different instances of the class.\n" +); + //a class held directly inside of another object class HeldClass : public Type { HeldClass() : @@ -353,7 +366,6 @@ class HeldClass : public Type { m_hasConvertFromMagicMethod(false), m_refToType(nullptr) { - m_needs_post_init = true; } public: @@ -404,12 +416,26 @@ class HeldClass : public Type { } } + const char* docConcrete() { + return HeldClass_doc; + } + void postInitializeConcrete() { - for (auto b: m_bases) { - b->postInitialize(); + // ensure our bases have their MROs initialized as well + for (auto h: m_bases) { + h->initializeMRO(); } initializeMRO(); + + size_t size = mBytesOfInitializationBits; + + for (auto t: m_members) { + m_byte_offsets.push_back(size); + size += t.getType()->bytecount(); + } + + m_size = size; } void initializeFromConcrete(Type* forwardDef) { @@ -917,8 +943,13 @@ class HeldClass : public Type { throw std::runtime_error("Makes no sense to initializeMRO on a forward"); } + if (m_mro.size()) { + return; + } + _computeMroSequence(); + for (size_t i = 0; i < m_mro.size(); i++) { m_ancestor_to_mro_index[m_mro[i]] = i; } @@ -989,17 +1020,9 @@ class HeldClass : public Type { setMagicMethodExistConstants(); - size_t size = mBytesOfInitializationBits; - - for (auto t: m_members) { - m_byte_offsets.push_back(size); - size += t.getType()->bytecount(); - } - m_is_default_constructible = ( m_memberFunctions.find("__init__") == m_memberFunctions.end() ); - m_size = size; } void updateBytesOfInitBits(); diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index d23d9559b..650c8f855 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -168,10 +168,6 @@ void OneOfType::updateInternalTypePointersConcrete( } void OneOfType::postInitializeConcrete() { - for (auto t: m_types) { - t->postInitialize(); - } - m_size = computeBytecount(); m_is_default_constructible = false; diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 61065cb64..3898954d8 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -29,8 +29,6 @@ class OneOfType : public Type { OneOfType() noexcept : Type(TypeCategory::catOneOf) { - m_needs_post_init = true; - m_doc = OneOf_doc; } // forward initialization @@ -39,10 +37,13 @@ class OneOfType : public Type { m_types(types) { m_is_forward_defined = true; - m_doc = OneOf_doc; } public: + const char* docConcrete() { + return OneOf_doc; + } + template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); diff --git a/typed_python/PointerToType.hpp b/typed_python/PointerToType.hpp index fa7bd7e81..d4db91bb0 100644 --- a/typed_python/PointerToType.hpp +++ b/typed_python/PointerToType.hpp @@ -44,8 +44,8 @@ class PointerTo : public Type { // construct a non-forward defined pointer PointerTo() : Type(TypeCategory::catPointerTo) { - m_doc = PointerTo_doc; - m_needs_post_init = true; + m_size = sizeof(instance); + m_is_default_constructible = true; } public: @@ -53,10 +53,13 @@ class PointerTo : public Type { Type(TypeCategory::catPointerTo), m_element_type(t) { + m_is_forward_defined = true; m_size = sizeof(instance); m_is_default_constructible = true; - m_doc = PointerTo_doc; - m_is_forward_defined = true; + } + + const char* docConcrete() { + return PointerTo_doc; } template diff --git a/typed_python/PyCellType.hpp b/typed_python/PyCellType.hpp index de6150dde..74fdf3ef2 100644 --- a/typed_python/PyCellType.hpp +++ b/typed_python/PyCellType.hpp @@ -19,6 +19,11 @@ #include "util.hpp" #include "Type.hpp" + +PyDoc_STRVAR(PyCell_doc, + "PyCell: represents a python-native cell instance.\n" +); + //wraps a python Cell class PyCellType : public PyObjectHandleTypeBase { public: @@ -28,7 +33,10 @@ class PyCellType : public PyObjectHandleTypeBase { m_name = std::string("PyCell"); m_size = sizeof(layout_type*); m_is_default_constructible = true; - m_is_simple = false; + } + + const char* docConcrete() { + return PyCell_doc; } template diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index 3a2ab9424..9555e728e 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -126,8 +126,6 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { m_name = std::string("PythonObjectOfType(") + mPyTypePtr->tp_name + ")"; - m_is_simple = false; - m_size = sizeof(layout_type*); int isinst = PyObject_IsInstance(Py_None, (PyObject*)mPyTypePtr); diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index e1b4c405d..e899eb1f7 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -25,7 +25,8 @@ class RefTo : public Type { // construct a non-forward defined refto RefTo() : Type(TypeCategory::catRefTo) { - m_needs_post_init = true; + m_size = sizeof(instance); + m_is_default_constructible = true; } public: diff --git a/typed_python/RegisterTypes.hpp b/typed_python/RegisterTypes.hpp index 376822ecf..bff20161e 100644 --- a/typed_python/RegisterTypes.hpp +++ b/typed_python/RegisterTypes.hpp @@ -191,8 +191,6 @@ class RegisterType : public Type { { m_size = sizeof(T); m_is_default_constructible = true; - m_is_forward_defined = false; - m_needs_post_init = false; } bool isPODConcrete() { @@ -285,7 +283,10 @@ class Bool : public RegisterType { Bool() : RegisterType(TypeCategory::catBool) { m_name = "bool"; - m_doc = Bool_doc; + } + + const char* docConcrete() { + return Bool_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -306,7 +307,10 @@ class UInt8 : public RegisterType { UInt8() : RegisterType(TypeCategory::catUInt8) { m_name = "UInt8"; - m_doc = UInt8_doc; + } + + const char* docConcrete() { + return UInt8_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -327,7 +331,10 @@ class UInt16 : public RegisterType { UInt16() : RegisterType(TypeCategory::catUInt16) { m_name = "UInt16"; - m_doc = UInt16_doc; + } + + const char* docConcrete() { + return UInt16_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -345,7 +352,10 @@ class UInt32 : public RegisterType { UInt32() : RegisterType(TypeCategory::catUInt32) { m_name = "UInt32"; - m_doc = UInt32_doc; + } + + const char* docConcrete() { + return UInt32_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -363,7 +373,10 @@ class UInt64 : public RegisterType { UInt64() : RegisterType(TypeCategory::catUInt64) { m_name = "UInt64"; - m_doc = UInt64_doc; + } + + const char* docConcrete() { + return UInt64_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -380,9 +393,12 @@ class Int8 : public RegisterType { public: Int8() : RegisterType(TypeCategory::catInt8) { - m_name = "Int8"; - m_doc = Int8_doc; m_size = 1; // Why only specify this here and not in other specializations? + m_name = "Int8"; + } + + const char* docConcrete() { + return Int8_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -400,7 +416,10 @@ class Int16 : public RegisterType { Int16() : RegisterType(TypeCategory::catInt16) { m_name = "Int16"; - m_doc = Int16_doc; + } + + const char* docConcrete() { + return Int16_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -418,7 +437,10 @@ class Int32 : public RegisterType { Int32() : RegisterType(TypeCategory::catInt32) { m_name = "Int32"; - m_doc = Int32_doc; + } + + const char* docConcrete() { + return Int32_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -436,7 +458,10 @@ class Int64 : public RegisterType { Int64() : RegisterType(TypeCategory::catInt64) { m_name = "int"; - m_doc = Int64_doc; + } + + const char* docConcrete() { + return Int64_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -454,7 +479,10 @@ class Float32 : public RegisterType { Float32() : RegisterType(TypeCategory::catFloat32) { m_name = "Float32"; - m_doc = Float32_doc; + } + + const char* docConcrete() { + return Float32_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { @@ -472,7 +500,10 @@ class Float64 : public RegisterType { Float64() : RegisterType(TypeCategory::catFloat64) { m_name = "float"; - m_doc = Float64_doc; + } + + const char* docConcrete() { + return Float64_doc; } void repr(instance_ptr self, ReprAccumulator& stream, bool isStr) { diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index 9c041b522..5ee629e0e 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -29,18 +29,19 @@ PyDoc_STRVAR(Set_doc, class SetType : public Type { SetType() : Type(TypeCategory::catSet) { - m_doc = Set_doc; - m_needs_post_init = true; } public: SetType(Type* eltype) : Type(TypeCategory::catSet) , m_key_type(eltype) { - m_doc = Set_doc; m_is_forward_defined = true; } + const char* docConcrete() { + return Set_doc; + } + template void _visitReferencedTypes(const visitor_type& visitor) { visitor(m_key_type); diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index 43bf9bcfd..7d52c539b 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -27,9 +27,7 @@ PyDoc_STRVAR(SubclassOf_doc, class SubclassOfType : public Type { SubclassOfType() : Type(TypeCategory::catSubclassOf) { - m_doc = SubclassOf_doc; m_is_default_constructible = true; - m_needs_post_init = true; m_size = sizeof(Type*); } public: @@ -37,12 +35,15 @@ class SubclassOfType : public Type { Type(TypeCategory::catSubclassOf), m_subclassOf(subclassOf) { - m_doc = SubclassOf_doc; m_is_forward_defined = true; m_is_default_constructible = true; m_size = sizeof(Type*); } + const char* docConcrete() { + return SubclassOf_doc; + } + void postInitializeConcrete() { } diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index a2cf76e56..e92f5ecf9 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -27,7 +27,6 @@ class TupleOrListOfType : public Type { m_element_type(nullptr), m_is_tuple(isTuple) { - m_needs_post_init = true; m_is_default_constructible = true; } @@ -627,7 +626,11 @@ class ListOfType : public TupleOrListOfType { public: ListOfType(Type* type) : TupleOrListOfType(type, false) { - m_doc = ListOf_doc; + } + + + const char* docConcrete() { + return ListOf_doc; } static ListOfType* Make(Type* elt); @@ -690,7 +693,10 @@ class TupleOfType : public TupleOrListOfType { // forward form TupleOfType(Type* type) : TupleOrListOfType(type, true) { - m_doc = TupleOf_doc; + } + + const char* docConcrete() { + return TupleOf_doc; } static TupleOfType* Make(Type* elt); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 5489f6bef..ee57cda1f 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -352,9 +352,25 @@ void Type::attemptToResolve() { typeAndSource.first->recomputeNamePostInitialization(); } - // cause each type to post-initialize itself, which lets it update internal bytecounts etc - for (auto typeAndSource: resolutionSource) { - typeAndSource.first->postInitialize(); + // cause each type to post-initialize itself, which lets it update internal bytecounts and + // default initialization flags + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto typeAndSource: resolutionSource) { + if (typeAndSource.first->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > resolutionSource.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } } // let each type update any internal caches it might need before it gets instantiated diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 73ce89655..2b4545937 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -342,8 +342,16 @@ class Type { return name(); } - const char* doc() const { - return m_doc; + const char* doc() { + return this->check([&](auto& subtype) { + return subtype.docConcrete(); + }); + } + + const char* docConcrete() { + throw std::runtime_error( + "No docstring provided for " + name() + " of category " + getTypeCategoryString() + ); } size_t bytecount() const { @@ -790,10 +798,6 @@ class Type { return m_is_default_constructible; } - bool isSimple() const { - return m_is_simple; - } - // are we guaranteed we can convert to this other type at the 'Signature' level bool canConvertToTrivially(Type* otherType); @@ -897,15 +901,12 @@ class Type { m_size(0), m_is_default_constructible(false), m_name("Undefined"), - m_doc(nullptr), mTypeRep(nullptr), m_base(nullptr), - m_is_simple(true), mTypeGroup(nullptr), mRecursiveTypeGroupIndex(-1), m_is_forward_defined(false), m_forward_resolves_to(nullptr), - m_needs_post_init(false), m_is_redundant(false) {} @@ -924,19 +925,12 @@ class Type { mutable std::string m_stripped_name; - const char* m_doc; - PyTypeObject* mTypeRep; Type* m_base; - // 'simple' types are those that have no reference to the python interpreter - bool m_is_simple; - enum BinaryCompatibilityCategory { Incompatible, Checking, Compatible }; - std::map mIsBinaryCompatible; - // a sha-hash that uniquely identifies this type. If this value is // the same for two types, then they should be indistinguishable except // for pointers values. @@ -959,9 +953,6 @@ class Type { // have we been made 'redundant'? bool m_is_redundant; - // do we need a post-initialization step? - bool m_needs_post_init; - // try to resolve this forward type. If we can't, we'll throw an exception. On exit, // we will have thrown, or m_forward_resolves_to will be populated. void attemptToResolve(); @@ -1036,16 +1027,16 @@ class Type { public: // finish initializing the type assuming no forward types are reachable - void postInitialize() { - if (!m_needs_post_init) { - return; - } - - m_needs_post_init = false; + bool postInitialize() { + size_t oldSize = m_size; + bool oldIsDefaultInit = m_is_default_constructible; this->check([&](auto& subtype) { subtype.postInitializeConcrete(); }); + + // return whether we changed + return m_size != oldSize || m_is_default_constructible != oldIsDefaultInit; } // update any caches in the type after we've walked over it @@ -1056,10 +1047,6 @@ class Type { } std::string computeRecursiveName(TypeStack& stack) { - if (!m_needs_post_init) { - return m_name; - } - long index = stack.indexOf(this); if (index != -1) { diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index 769b6cf94..eabdb6ca9 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -23,7 +23,6 @@ class TypedCellType : public Type { TypedCellType() : Type(TypeCategory::catTypedCell) { - m_needs_post_init = true; } public: class layout { @@ -41,8 +40,6 @@ class TypedCellType : public Type { { m_is_forward_defined = true; - m_is_simple = false; - m_size = sizeof(layout*); m_is_default_constructible = true; diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 547628d64..1161e16a8 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -20,10 +20,8 @@ Value::Value() : Type(TypeCategory::catValue), mValueAsPyobj(nullptr) { - m_needs_post_init = true; m_size = 0; m_is_default_constructible = true; - m_doc = Value_doc; } @@ -35,7 +33,6 @@ Value::Value(const Instance& instance) : m_size = 0; m_is_default_constructible = true; - m_doc = Value_doc; mValueAsPyobj = PyInstance::extractPythonObject(mInstance); m_is_forward_defined = true; } diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index a9af8545f..4940847fb 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -24,6 +24,10 @@ PyDoc_STRVAR(Value_doc, class Value : public Type { public: + const char* docConcrete() { + return Value_doc; + } + template void _visitContainedTypes(const visitor_type& visitor) { } diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 5db8d458f..4efea9bc4 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2554,23 +2554,6 @@ PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { }); } -PyObject *isSimple(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "isSimple takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); - - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); - - if (!t) { - PyErr_SetString(PyExc_TypeError, "first argument to 'isSimple' must be a type object"); - return NULL; - } - - return incref(t->isSimple() ? Py_True : Py_False); -} - PyObject *isPOD(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "isPOD takes 1 positional argument"); @@ -3488,7 +3471,6 @@ static PyMethodDef module_methods[] = { {"buildCodeObject", (PyCFunction)buildCodeObject, METH_VARARGS | METH_KEYWORDS, NULL}, {"buildPyFunctionObject", (PyCFunction)buildPyFunctionObject, METH_VARARGS | METH_KEYWORDS, NULL}, {"isPOD", (PyCFunction)isPOD, METH_VARARGS, NULL}, - {"isSimple", (PyCFunction)isSimple, METH_VARARGS, NULL}, {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/typed_python/compiler/python_object_representation.py b/typed_python/compiler/python_object_representation.py index adb202c8b..07780505d 100644 --- a/typed_python/compiler/python_object_representation.py +++ b/typed_python/compiler/python_object_representation.py @@ -26,7 +26,6 @@ from typed_python.compiler.type_wrappers.python_type_object_wrapper import PythonTypeObjectWrapper from typed_python.compiler.type_wrappers.module_wrapper import ModuleWrapper from typed_python.compiler.type_wrappers.typed_cell_wrapper import TypedCellWrapper -from typed_python.compiler.type_wrappers.unresolved_forward_type_wrapper import UnresolvedForwardTypeWrapper from typed_python.compiler.type_wrappers.python_free_function_wrapper import PythonFreeFunctionWrapper from typed_python.compiler.type_wrappers.python_free_object_wrapper import PythonFreeObjectWrapper from typed_python.compiler.type_wrappers.python_typed_function_wrapper import PythonTypedFunctionWrapper @@ -78,7 +77,7 @@ from types import ModuleType from typed_python._types import ( TypeFor, bytecount, prepareArgumentToBePassedToCompiler, - serialize, deserialize + serialize, deserialize, isForwardDefined ) from typed_python import ( Type, Int32, Int16, Int8, UInt64, UInt32, UInt16, @@ -152,7 +151,7 @@ def _typedPythonTypeToTypeWrapper(t): assert isinstance(t, type), t - if isinstance(t, Forward): + if isForwardDefined(t): raise Exception("Can't compile against Forward types") if not hasattr(t, '__typed_python_category__'): diff --git a/typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py b/typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py deleted file mode 100644 index fdabb7730..000000000 --- a/typed_python/compiler/type_wrappers/unresolved_forward_type_wrapper.py +++ /dev/null @@ -1,131 +0,0 @@ -from typed_python.compiler.type_wrappers.wrapper import Wrapper -import typed_python.compiler.native_compiler.native_ast as native_ast -from typed_python.compiler.conversion_level import ConversionLevel - - -class UnresolvedForwardTypeWrapper(Wrapper): - def __init__(self, t): - super().__init__(t) - - def getNativeLayoutType(self): - return native_ast.Void - - def throwUnresolvedForwardException(self, context): - return context.pushException( - TypeError, - f"Type {self.typeRepresentation.__name__} has unresolved forwards" - ) - - def convert_incref(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_fastnext(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_attribute_pointerTo(self, context, pointerInstance, attribute): - return self.throwUnresolvedForwardException(context) - - def convert_attribute(self, context, instance, attribute): - return self.throwUnresolvedForwardException(context) - - def convert_set_attribute(self, context, instance, attribute, value): - return self.throwUnresolvedForwardException(context) - - def convert_delitem(self, context, instance, item): - return self.throwUnresolvedForwardException(context) - - def convert_getitem(self, context, instance, item): - return self.throwUnresolvedForwardException(context) - - def convert_getslice(self, context, instance, start, stop, step): - return self.throwUnresolvedForwardException(context) - - def convert_setitem(self, context, instance, index, value): - return self.throwUnresolvedForwardException(context) - - def convert_assign(self, context, target, toStore): - return self.throwUnresolvedForwardException(context) - - def convert_copy_initialize(self, context, target, toStore): - return self.throwUnresolvedForwardException(context) - - def convert_destroy(self, context, instance): - return self.throwUnresolvedForwardException(context) - - def convert_default_initialize(self, context, target): - return self.throwUnresolvedForwardException(context) - - def convert_typeof(self, context, instance): - return self.throwUnresolvedForwardException(context) - - def convert_issubclass(self, context, instance, ofType, isSubclassCall): - return self.throwUnresolvedForwardException(context) - - def convert_masquerade_to_untyped(self, context, instance): - return self.throwUnresolvedForwardException(context) - - def convert_call(self, context, left, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def convert_len(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_hash(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_abs(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_index_cast(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_math_float_cast(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_builtin(self, f, context, expr, a1=None): - return self.throwUnresolvedForwardException(context) - - def convert_repr(self, context, expr): - return self.throwUnresolvedForwardException(context) - - def convert_unary_op(self, context, expr, op): - return self.throwUnresolvedForwardException(context) - - def convert_to_type(self, context, expr, target_type, level: ConversionLevel, assumeSuccessful=False): - return self.throwUnresolvedForwardException(context) - - def convert_to_type_with_target(self, context, expr, targetVal, level: ConversionLevel, mayThrowOnFailure=False): - return self.throwUnresolvedForwardException(context) - - def convert_to_self_with_target(self, context, targetVal, sourceVal, level: ConversionLevel, mayThrowOnFailure=False): - return self.throwUnresolvedForwardException(context) - - def convert_bin_op(self, context, l, op, r, inplace): - return self.throwUnresolvedForwardException(context) - - def convert_bin_op_reverse(self, context, r, op, l, inplace): - return self.throwUnresolvedForwardException(context) - - def convert_format(self, context, instance, formatSpecOrNone=None): - return self.throwUnresolvedForwardException(context) - - def convert_type_attribute(self, context, typeInst, attribute): - return self.throwUnresolvedForwardException(context) - - def convert_type_call(self, context, typeInst, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def convert_call_on_container_expression(self, context, inst, argExpr): - return self.throwUnresolvedForwardException(context) - - def convert_type_call_on_container_expression(self, context, typeInst, argExpr): - return self.throwUnresolvedForwardException(context) - - def convert_type_method_call(self, context, typeInst, methodname, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def convert_method_call(self, context, instance, methodname, args, kwargs): - return self.throwUnresolvedForwardException(context) - - def get_iteration_expressions(self, context, expr): - return self.throwUnresolvedForwardException(context) diff --git a/typed_python/python_ast.py b/typed_python/python_ast.py index 49fbccd1c..e5037152f 100644 --- a/typed_python/python_ast.py +++ b/typed_python/python_ast.py @@ -24,7 +24,7 @@ import types import traceback -from typed_python._types import Forward, Alternative, TupleOf, OneOf +from typed_python._types import Forward, Alternative, TupleOf, OneOf, resolveForwardDefinedType # forward declarations. @@ -46,7 +46,7 @@ WithItem = Forward("WithItem") TypeIgnore = Forward("TypeIgnore") -Module = Module.define(Alternative( +Module.define(Alternative( "Module", Module={ "body": TupleOf(Statement), @@ -57,7 +57,7 @@ Suite={"body": TupleOf(Statement)} )) -TypeIgnore = TypeIgnore.define(Alternative( +TypeIgnore.define(Alternative( "TypeIgnore", Item={'lineno': int, 'tag': str} )) @@ -161,7 +161,7 @@ def StatementStr(self): return "\n".join(list(statementStrLines(self))) -Statement = Statement.define(Alternative( +Statement.define(Alternative( "Statement", FunctionDef={ "name": str, @@ -475,7 +475,7 @@ def ExpressionStr(self): return str(type(self)) -Expr = Expr.define(Alternative( +Expr.define(Alternative( "Expr", BoolOp={ "op": BooleanOp, @@ -684,7 +684,7 @@ def ExpressionStr(self): __str__=ExpressionStr )) -NumericConstant = NumericConstant.define(Alternative( +NumericConstant.define(Alternative( "NumericConstant", Int={"value": int}, Long={"value": str}, @@ -702,7 +702,7 @@ def ExpressionStr(self): ) )) -ExprContext = ExprContext.define(Alternative( +ExprContext.define(Alternative( "ExprContext", Load={}, Store={}, @@ -712,13 +712,13 @@ def ExpressionStr(self): Param={} )) -BooleanOp = BooleanOp.define(Alternative( +BooleanOp.define(Alternative( "BooleanOp", And={}, Or={} )) -BinaryOp = BinaryOp.define(Alternative( +BinaryOp.define(Alternative( "BinaryOp", Add={}, Sub={}, @@ -735,7 +735,7 @@ def ExpressionStr(self): MatMult={} )) -UnaryOp = UnaryOp.define(Alternative( +UnaryOp.define(Alternative( "UnaryOp", Invert={}, Not={}, @@ -743,7 +743,7 @@ def ExpressionStr(self): USub={} )) -ComparisonOp = ComparisonOp.define(Alternative( +ComparisonOp.define(Alternative( "ComparisonOp", Eq={}, NotEq={}, @@ -757,7 +757,7 @@ def ExpressionStr(self): NotIn={} )) -Comprehension = Comprehension.define(Alternative( +Comprehension.define(Alternative( "Comprehension", Item={ "target": Expr, @@ -767,7 +767,7 @@ def ExpressionStr(self): } )) -ExceptionHandler = ExceptionHandler.define(Alternative( +ExceptionHandler.define(Alternative( "ExceptionHandler", Item={ "type": OneOf(Expr, None), @@ -779,7 +779,7 @@ def ExpressionStr(self): } )) -Arguments = Arguments.define(Alternative( +Arguments.define(Alternative( "Arguments", Item={ **({'posonlyargs': TupleOf(Arg)} if sys.version_info.minor >= 8 else {}), @@ -802,7 +802,7 @@ def ExpressionStr(self): + ([self.kwarg.arg] if self.kwarg else []) )) -Arg = Arg.define(Alternative( +Arg.define(Alternative( "Arg", Item={ 'arg': str, @@ -813,7 +813,7 @@ def ExpressionStr(self): } )) -Keyword = Keyword.define(Alternative( +Keyword.define(Alternative( "Keyword", Item={ "arg": OneOf(None, str), @@ -822,7 +822,7 @@ def ExpressionStr(self): } )) -Alias = Alias.define(Alternative( +Alias.define(Alternative( "Alias", Item={ "name": str, @@ -835,7 +835,7 @@ def ExpressionStr(self): } )) -WithItem = WithItem.define(Alternative( +WithItem.define(Alternative( "WithItem", Item={ "context_expr": Expr, @@ -886,123 +886,6 @@ def makeExtSlice(dims): return Expr.Tuple(elts=dims) -# map Python AST types to our syntax-tree types (defined `ove) -converters = { - ast.Module: Module.Module, - ast.Expression: Module.Expression, - ast.Interactive: Module.Interactive, - ast.Suite: Module.Suite, - ast.FunctionDef: Statement.FunctionDef, - ast.ClassDef: Statement.ClassDef, - ast.Return: Statement.Return, - ast.Delete: Statement.Delete, - ast.Assign: Statement.Assign, - ast.AugAssign: Statement.AugAssign, - ast.AnnAssign: Statement.AnnAssign, - ast.For: Statement.For, - ast.While: Statement.While, - ast.If: Statement.If, - ast.With: Statement.With, - ast.Raise: Statement.Raise, - ast.Try: Statement.Try, - ast.Assert: Statement.Assert, - ast.Import: Statement.Import, - ast.ImportFrom: Statement.ImportFrom, - ast.Global: Statement.Global, - ast.Nonlocal: Statement.NonLocal, - ast.Expr: Statement.Expr, - ast.Pass: Statement.Pass, - ast.Break: Statement.Break, - ast.Continue: Statement.Continue, - ast.BoolOp: Expr.BoolOp, - ast.BinOp: Expr.BinOp, - ast.UnaryOp: Expr.UnaryOp, - ast.Lambda: Expr.Lambda, - ast.IfExp: Expr.IfExp, - ast.Dict: Expr.Dict, - ast.Set: Expr.Set, - ast.JoinedStr: Expr.JoinedStr, - ast.Bytes: Expr.Bytes, - ast.Constant: Expr.Constant, - ast.FormattedValue: Expr.FormattedValue, - ast.ListComp: Expr.ListComp, - ast.AsyncFunctionDef: Statement.AsyncFunctionDef, - ast.AsyncWith: Statement.AsyncWith, - ast.AsyncFor: Statement.AsyncFor, - ast.Await: Expr.Await, - ast.SetComp: Expr.SetComp, - ast.DictComp: Expr.DictComp, - ast.GeneratorExp: Expr.GeneratorExp, - ast.Yield: Expr.Yield, - ast.YieldFrom: Expr.YieldFrom, - ast.Compare: Expr.Compare, - ast.Call: Expr.Call, - ast.Num: createPythonAstConstant, - ast.Str: createPythonAstString, - ast.Attribute: Expr.Attribute, - ast.Subscript: Expr.Subscript, - ast.Name: Expr.Name, - ast.NameConstant: makeNameConstant, - ast.List: Expr.List, - ast.Tuple: Expr.Tuple, - ast.Starred: Expr.Starred, - ast.Load: ExprContext.Load, - ast.Store: ExprContext.Store, - ast.Del: ExprContext.Del, - ast.AugLoad: ExprContext.AugLoad, - ast.AugStore: ExprContext.AugStore, - ast.Param: ExprContext.Param, - ast.Ellipsis: makeEllipsis, - ast.Slice: Expr.Slice, - ast.ExtSlice: makeExtSlice, - ast.Index: lambda value: value, - ast.And: BooleanOp.And, - ast.Or: BooleanOp.Or, - ast.Add: BinaryOp.Add, - ast.Sub: BinaryOp.Sub, - ast.Mult: BinaryOp.Mult, - ast.MatMult: BinaryOp.MatMult, - ast.Div: BinaryOp.Div, - ast.Mod: BinaryOp.Mod, - ast.Pow: BinaryOp.Pow, - ast.LShift: BinaryOp.LShift, - ast.RShift: BinaryOp.RShift, - ast.BitOr: BinaryOp.BitOr, - ast.BitXor: BinaryOp.BitXor, - ast.BitAnd: BinaryOp.BitAnd, - ast.FloorDiv: BinaryOp.FloorDiv, - ast.Invert: UnaryOp.Invert, - ast.Not: UnaryOp.Not, - ast.UAdd: UnaryOp.UAdd, - ast.USub: UnaryOp.USub, - ast.Eq: ComparisonOp.Eq, - ast.NotEq: ComparisonOp.NotEq, - ast.Lt: ComparisonOp.Lt, - ast.LtE: ComparisonOp.LtE, - ast.Gt: ComparisonOp.Gt, - ast.GtE: ComparisonOp.GtE, - ast.Is: ComparisonOp.Is, - ast.IsNot: ComparisonOp.IsNot, - ast.In: ComparisonOp.In, - ast.NotIn: ComparisonOp.NotIn, - ast.comprehension: Comprehension.Item, - ast.excepthandler: lambda x: x, - ast.ExceptHandler: ExceptionHandler.Item, - ast.arguments: Arguments.Item, - ast.arg: Arg.Item, - ast.keyword: Keyword.Item, - ast.alias: Alias.Item, - ast.withitem: WithItem.Item, - **({'ast.type_ignore': TypeIgnore.Item} if sys.version_info.minor >= 8 else {}), -} - -# most converters map to an alternative type -reverseConverters = { - t: v for v, t in converters.items() - if hasattr(t, '__typed_python_category__') and t.__typed_python_category__ == "ConcreteAlternative" -} - - def convertAlgebraicArgs(pyAst, *members): members = [x for x in members if x not in ['line_number', 'col_offset']] return {m: convertAlgebraicToPyAst(getattr(pyAst, m)) for m in members} @@ -1501,3 +1384,140 @@ def evaluateFunctionDefWithLocalsInCells(pyAst, globals, locals, stripAnnotation cacheAstForCode(inner.__code__, pyAst) return inner + + +Module = resolveForwardDefinedType(Module) +Statement = resolveForwardDefinedType(Statement) +Expr = resolveForwardDefinedType(Expr) +Arg = resolveForwardDefinedType(Arg) +NumericConstant = resolveForwardDefinedType(NumericConstant) +ExprContext = resolveForwardDefinedType(ExprContext) +BooleanOp = resolveForwardDefinedType(BooleanOp) +BinaryOp = resolveForwardDefinedType(BinaryOp) +UnaryOp = resolveForwardDefinedType(UnaryOp) +ComparisonOp = resolveForwardDefinedType(ComparisonOp) +Comprehension = resolveForwardDefinedType(Comprehension) +ExceptionHandler = resolveForwardDefinedType(ExceptionHandler) +Arguments = resolveForwardDefinedType(Arguments) +Keyword = resolveForwardDefinedType(Keyword) +Alias = resolveForwardDefinedType(Alias) +WithItem = resolveForwardDefinedType(WithItem) +TypeIgnore = resolveForwardDefinedType(TypeIgnore) + + + +# map Python AST types to our syntax-tree types +converters = { + ast.Module: Module.Module, + ast.Expression: Module.Expression, + ast.Interactive: Module.Interactive, + ast.Suite: Module.Suite, + ast.FunctionDef: Statement.FunctionDef, + ast.ClassDef: Statement.ClassDef, + ast.Return: Statement.Return, + ast.Delete: Statement.Delete, + ast.Assign: Statement.Assign, + ast.AugAssign: Statement.AugAssign, + ast.AnnAssign: Statement.AnnAssign, + ast.For: Statement.For, + ast.While: Statement.While, + ast.If: Statement.If, + ast.With: Statement.With, + ast.Raise: Statement.Raise, + ast.Try: Statement.Try, + ast.Assert: Statement.Assert, + ast.Import: Statement.Import, + ast.ImportFrom: Statement.ImportFrom, + ast.Global: Statement.Global, + ast.Nonlocal: Statement.NonLocal, + ast.Expr: Statement.Expr, + ast.Pass: Statement.Pass, + ast.Break: Statement.Break, + ast.Continue: Statement.Continue, + ast.BoolOp: Expr.BoolOp, + ast.BinOp: Expr.BinOp, + ast.UnaryOp: Expr.UnaryOp, + ast.Lambda: Expr.Lambda, + ast.IfExp: Expr.IfExp, + ast.Dict: Expr.Dict, + ast.Set: Expr.Set, + ast.JoinedStr: Expr.JoinedStr, + ast.Bytes: Expr.Bytes, + ast.Constant: Expr.Constant, + ast.FormattedValue: Expr.FormattedValue, + ast.ListComp: Expr.ListComp, + ast.AsyncFunctionDef: Statement.AsyncFunctionDef, + ast.AsyncWith: Statement.AsyncWith, + ast.AsyncFor: Statement.AsyncFor, + ast.Await: Expr.Await, + ast.SetComp: Expr.SetComp, + ast.DictComp: Expr.DictComp, + ast.GeneratorExp: Expr.GeneratorExp, + ast.Yield: Expr.Yield, + ast.YieldFrom: Expr.YieldFrom, + ast.Compare: Expr.Compare, + ast.Call: Expr.Call, + ast.Num: createPythonAstConstant, + ast.Str: createPythonAstString, + ast.Attribute: Expr.Attribute, + ast.Subscript: Expr.Subscript, + ast.Name: Expr.Name, + ast.NameConstant: makeNameConstant, + ast.List: Expr.List, + ast.Tuple: Expr.Tuple, + ast.Starred: Expr.Starred, + ast.Load: ExprContext.Load, + ast.Store: ExprContext.Store, + ast.Del: ExprContext.Del, + ast.AugLoad: ExprContext.AugLoad, + ast.AugStore: ExprContext.AugStore, + ast.Param: ExprContext.Param, + ast.Ellipsis: makeEllipsis, + ast.Slice: Expr.Slice, + ast.ExtSlice: makeExtSlice, + ast.Index: lambda value: value, + ast.And: BooleanOp.And, + ast.Or: BooleanOp.Or, + ast.Add: BinaryOp.Add, + ast.Sub: BinaryOp.Sub, + ast.Mult: BinaryOp.Mult, + ast.MatMult: BinaryOp.MatMult, + ast.Div: BinaryOp.Div, + ast.Mod: BinaryOp.Mod, + ast.Pow: BinaryOp.Pow, + ast.LShift: BinaryOp.LShift, + ast.RShift: BinaryOp.RShift, + ast.BitOr: BinaryOp.BitOr, + ast.BitXor: BinaryOp.BitXor, + ast.BitAnd: BinaryOp.BitAnd, + ast.FloorDiv: BinaryOp.FloorDiv, + ast.Invert: UnaryOp.Invert, + ast.Not: UnaryOp.Not, + ast.UAdd: UnaryOp.UAdd, + ast.USub: UnaryOp.USub, + ast.Eq: ComparisonOp.Eq, + ast.NotEq: ComparisonOp.NotEq, + ast.Lt: ComparisonOp.Lt, + ast.LtE: ComparisonOp.LtE, + ast.Gt: ComparisonOp.Gt, + ast.GtE: ComparisonOp.GtE, + ast.Is: ComparisonOp.Is, + ast.IsNot: ComparisonOp.IsNot, + ast.In: ComparisonOp.In, + ast.NotIn: ComparisonOp.NotIn, + ast.comprehension: Comprehension.Item, + ast.excepthandler: lambda x: x, + ast.ExceptHandler: ExceptionHandler.Item, + ast.arguments: Arguments.Item, + ast.arg: Arg.Item, + ast.keyword: Keyword.Item, + ast.alias: Alias.Item, + ast.withitem: WithItem.Item, + **({'ast.type_ignore': TypeIgnore.Item} if sys.version_info.minor >= 8 else {}), +} + +# most converters map to an alternative type +reverseConverters = { + t: v for v, t in converters.items() + if hasattr(t, '__typed_python_category__') and t.__typed_python_category__ == "ConcreteAlternative" +} diff --git a/typed_python/types_test.py b/typed_python/types_test.py index 11537fb44..0e8142b8d 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -341,31 +341,6 @@ def test_one_of_alternative(self): self.assertEqual(Ox(X.V(a=10)), X.V(a=10)) - def test_one_of_py_subclass(self): - class X(NamedTuple(x=int)): - def f(self): - return self.x - - Ox = OneOf(None, X) - - self.assertEqual(NamedTuple(x=int)(x=10).x, 10) - self.assertEqual(X(x=10).f(), 10) - self.assertEqual(Ox(X(x=10)).f(), 10) - - def test_one_of_distinguishes_py_subclasses(self): - class X(NamedTuple(x=int)): - def f(self): - return self.x - - class X2(NamedTuple(x=int)): - def f(self): - return self.x + 2 - - XorX2 = OneOf(X2, X) - - self.assertTrue(isinstance(XorX2(X()), X)) - self.assertTrue(isinstance(XorX2(X2()), X2)) - def test_function_as_type_arg(self): @Function def f(x: int): @@ -2353,57 +2328,6 @@ def test_mutable_dict_iteration_order(self): del d[1] self.assertEqual(list(d), [10, 2]) - def test_simplicity(self): - isSimple = _types.isSimple - - self.assertTrue(isSimple(int)) - self.assertTrue(isSimple(Int32())) - self.assertTrue(isSimple(Int16())) - self.assertTrue(isSimple(Int8())) - self.assertTrue(isSimple(UInt64())) - self.assertTrue(isSimple(UInt32())) - self.assertTrue(isSimple(UInt16())) - self.assertTrue(isSimple(UInt8())) - self.assertTrue(isSimple(str)) - self.assertTrue(isSimple(bytes)) - self.assertTrue(isSimple(bool)) - self.assertTrue(isSimple(float)) - - class C(Class): - pass - - self.assertFalse(isSimple(C)) - - self.assertTrue(isSimple(ListOf(int))) - self.assertFalse(isSimple(ListOf(C))) - - self.assertTrue(isSimple(TupleOf(int))) - self.assertFalse(isSimple(TupleOf(C))) - - self.assertTrue(isSimple(ConstDict(int, int))) - self.assertFalse(isSimple(ConstDict(C, int))) - self.assertFalse(isSimple(ConstDict(int, C))) - - self.assertTrue(isSimple(Dict(int, int))) - self.assertFalse(isSimple(Dict(C, int))) - self.assertFalse(isSimple(Dict(int, C))) - - self.assertTrue(isSimple(Set(int))) - self.assertFalse(isSimple(Set(C))) - - self.assertFalse(isSimple(Alternative("Alternative"))) - - self.assertTrue(isSimple(NamedTuple(x=int))) - self.assertFalse(isSimple(NamedTuple(x=C))) - - X = Forward("X") - X = X.define(Alternative("X", X={'x': X}, Y={'i': int})) - self.assertFalse(isSimple(X)) - self.assertFalse(isSimple(NamedTuple(x=X))) - - self.assertFalse(isSimple(OneOf(int, X))) - self.assertTrue(isSimple(OneOf(int, float))) - def test_oneof_picks_best_choice(self): T = OneOf(float, int, bool) From 8a70c16caa3910f7b82fb85ee6858b4b4e8b9329 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 7 Jun 2023 19:30:51 +0000 Subject: [PATCH 23/83] Types serialization test will run (even though it crashes) --- typed_python/types_serialization_test.py | 55 ------------------------ 1 file changed, 55 deletions(-) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 5514a735c..69150abf9 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -84,11 +84,6 @@ def method(self): pass -class ModuleLevelNamedTupleSubclass(NamedTuple(x=int)): - def f(self): - return self.x - - class ModuleLevelClass(Class, Final): def f(self): return "HI!" @@ -1877,56 +1872,6 @@ def test_serialize_module_level_class(self): sc.serialize(ModuleLevelClass), ) - def test_serialize_unnamed_subclass_of_named_tuple(self): - class SomeNamedTuple(NamedTuple(x=int)): - def f(self): - return self.x - - sc = SerializationContext() - - self.assertEqual( - sc.deserialize(sc.serialize(SomeNamedTuple))(x=10).f(), - 10 - ) - - self.assertEqual( - sc.deserialize(sc.serialize(SomeNamedTuple(x=10))).f(), - 10 - ) - - def test_serialize_named_subclass_of_named_tuple(self): - sc = SerializationContext() - - self.assertEqual( - ModuleLevelNamedTupleSubclass.__module__, - "typed_python.types_serialization_test" - ) - - self.assertEqual( - ModuleLevelNamedTupleSubclass.__name__, - "ModuleLevelNamedTupleSubclass" - ) - - self.assertIs( - sc.deserialize(sc.serialize(ModuleLevelNamedTupleSubclass)), - ModuleLevelNamedTupleSubclass - ) - - self.assertIs( - type(sc.deserialize(sc.serialize(ModuleLevelNamedTupleSubclass()))), - ModuleLevelNamedTupleSubclass - ) - - self.assertIs( - type(sc.deserialize(sc.serialize([ModuleLevelNamedTupleSubclass()]))[0]), - ModuleLevelNamedTupleSubclass - ) - - self.assertIs( - sc.deserialize(sc.serialize(ModuleLevelNamedTupleSubclass(x=10))).f(), - 10 - ) - def test_serialize_builtin_tp_functions(self): sc = SerializationContext() From 9c4a2988a6fef5165ce9c77e536b00e093463812 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 7 Jun 2023 22:42:21 +0000 Subject: [PATCH 24/83] Start to work out serialization. --- typed_python/AlternativeMatcherType.hpp | 3 +- typed_python/AlternativeType.hpp | 8 +- typed_python/BoundMethodType.hpp | 3 +- typed_python/ClassType.hpp | 9 +- typed_python/ConcreteAlternativeType.hpp | 6 +- typed_python/ConstDictType.hpp | 8 +- typed_python/DictType.hpp | 3 +- typed_python/HeldClassType.hpp | 2 +- typed_python/OneOfType.hpp | 2 +- typed_python/PointerToType.hpp | 4 +- typed_python/PythonObjectOfTypeType.hpp | 22 +++ typed_python/PythonSerializationContext.hpp | 2 + ...onSerializationContext_deserialization.cpp | 158 ++++++++++++++++-- typed_python/RefToType.hpp | 3 +- typed_python/SetType.hpp | 3 +- typed_python/SubclassOfType.hpp | 3 +- typed_python/TupleOrListOfType.hpp | 25 ++- typed_python/Type.hpp | 29 ++-- typed_python/TypedCellType.hpp | 7 +- typed_python/ValueType.hpp | 11 +- 20 files changed, 224 insertions(+), 87 deletions(-) diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index d83b2db53..3f06f04b5 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -26,10 +26,11 @@ PyDoc_STRVAR(AlternativeMatcher_doc, // Models an alternatives '.matches' result class AlternativeMatcher : public Type { +public: AlternativeMatcher() : Type(TypeCategory::catAlternativeMatcher) { } -public: + AlternativeMatcher(Type* inAlternative) : Type(TypeCategory::catAlternativeMatcher) { m_is_forward_defined = true; diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 9ad16b582..b3e55da26 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -77,10 +77,6 @@ PyDoc_STRVAR(Alternative_doc, ); class Alternative : public Type { - Alternative() : Type(TypeCategory::catAlternative) - { - } - public: class layout { public: @@ -92,6 +88,10 @@ class Alternative : public Type { typedef layout* layout_ptr; + Alternative() : Type(TypeCategory::catAlternative) + { + } + Alternative(std::string name, std::string moduleName, const std::vector >& subtypes, diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index 5e64d311b..36e9d60bc 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -26,10 +26,11 @@ PyDoc_STRVAR(BoundMethod_doc, class BoundMethod : public Type { +public: BoundMethod() : Type(TypeCategory::catBoundMethod) { } -public: + BoundMethod(Type* inFirstArg, std::string funcName) : Type(TypeCategory::catBoundMethod) { m_is_forward_defined = true; diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index e5dd715af..400ce02d0 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -35,11 +35,6 @@ PyDoc_STRVAR(Class_doc, ); class Class : public Type { - // this is the non-forward type resolution version of this - Class() : Type(catClass) - { - } - public: class layout { public: @@ -50,6 +45,10 @@ class Class : public Type { typedef layout* layout_ptr; + Class() : Type(catClass) + { + } + Class(std::string name, HeldClass* inClass) : Type(catClass), m_heldClass(inClass) diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index 96192593d..359d90fc0 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -20,6 +20,9 @@ #include "AlternativeType.hpp" class ConcreteAlternative : public Type { +public: + typedef Alternative::layout layout; + ConcreteAlternative() : Type(TypeCategory::catConcreteAlternative), m_which(0), @@ -27,9 +30,6 @@ class ConcreteAlternative : public Type { { } -public: - typedef Alternative::layout layout; - ConcreteAlternative(Alternative* m_alternative, int64_t which) : Type(TypeCategory::catConcreteAlternative), m_alternative(m_alternative), diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 252c39675..123d520d7 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -27,10 +27,6 @@ PyDoc_STRVAR(ConstDictType_doc, ); class ConstDictType : public Type { - ConstDictType() : Type(TypeCategory::catConstDict) - { - } - public: class layout { public: @@ -44,6 +40,10 @@ class ConstDictType : public Type { typedef layout* layout_ptr; + ConstDictType() : Type(TypeCategory::catConstDict) + { + } + ConstDictType(Type* key, Type* value) : Type(TypeCategory::catConstDict), m_key(key), diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index 95605f7bb..f9cfdc107 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -30,11 +30,12 @@ PyDoc_STRVAR(DictType_doc, ); class DictType : public Type { +public: // construct a non-forward uninitialized dict DictType() : Type(TypeCategory::catDict) { } -public: + // declare a forward-defined Dict DictType(Type* key, Type* value) : Type(TypeCategory::catDict), diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 22da68b42..298089ccd 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -352,6 +352,7 @@ PyDoc_STRVAR(HeldClass_doc, //a class held directly inside of another object class HeldClass : public Type { +public: HeldClass() : Type(catHeldClass), m_vtable(new VTable(this)), @@ -368,7 +369,6 @@ class HeldClass : public Type { { } -public: HeldClass(std::string inName, const std::vector& baseClasses, bool isFinal, diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 3898954d8..987c295d5 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -25,6 +25,7 @@ PyDoc_STRVAR(OneOf_doc, ); class OneOfType : public Type { +public: // clone initialization OneOfType() noexcept : Type(TypeCategory::catOneOf) @@ -39,7 +40,6 @@ class OneOfType : public Type { m_is_forward_defined = true; } -public: const char* docConcrete() { return OneOf_doc; } diff --git a/typed_python/PointerToType.hpp b/typed_python/PointerToType.hpp index d4db91bb0..8457280ed 100644 --- a/typed_python/PointerToType.hpp +++ b/typed_python/PointerToType.hpp @@ -38,7 +38,7 @@ PyDoc_STRVAR(PointerTo_doc, ); class PointerTo : public Type { -protected: +public: typedef void* instance; // construct a non-forward defined pointer @@ -48,7 +48,6 @@ class PointerTo : public Type { m_is_default_constructible = true; } -public: PointerTo(Type* t) : Type(TypeCategory::catPointerTo), m_element_type(t) @@ -101,7 +100,6 @@ class PointerTo : public Type { m_element_type = ((PointerTo*)forwardDefinitionOfSelf)->m_element_type; } - void updateInternalTypePointersConcrete(const std::map& groupMap) { auto it = groupMap.find(m_element_type); if (it != groupMap.end()) { diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index 9555e728e..fed88882f 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -113,6 +113,11 @@ class PyObjectHandleTypeBase : public Type { //refcount when we completely release a handle to a python object). class PythonObjectOfType : public PyObjectHandleTypeBase { public: + // clone pathway + PythonObjectOfType() : PyObjectHandleTypeBase(TypeCategory::catPythonObjectOfType) + { + } + PythonObjectOfType(PyTypeObject* typePtr, PyObject* givenType) : PyObjectHandleTypeBase(TypeCategory::catPythonObjectOfType) { @@ -135,6 +140,23 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { } m_is_default_constructible = isinst != 0; + m_is_forward_defined = true; + } + + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { + mPyTypePtr = ((PythonObjectOfType*)forwardDefinitionOfSelf)->mPyTypePtr; + mGivenType = ((PythonObjectOfType*)forwardDefinitionOfSelf)->mGivenType; + m_name = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_name; + m_size = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_size; + } + + void updateInternalTypePointersConcrete( + const std::map& groupMap + ) { + } + + Type* cloneForForwardResolutionConcrete() { + return new PythonObjectOfType(); } template diff --git a/typed_python/PythonSerializationContext.hpp b/typed_python/PythonSerializationContext.hpp index f36c87932..1964b0822 100644 --- a/typed_python/PythonSerializationContext.hpp +++ b/typed_python/PythonSerializationContext.hpp @@ -163,6 +163,8 @@ class PythonSerializationContext : public SerializationContext { void deserializeClassClassMemberDict(std::map& dict, DeserializationBuffer& b, int wireType) const; + static Type* constructBlankInstanceOfType(Type::TypeCategory typeCat); + Type* deserializeNativeTypeInner(DeserializationBuffer& b, size_t wireType, bool actuallyProduceResult) const; Instance deserializeNativeInstance(DeserializationBuffer& b, size_t wireType) const; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index fca8c7912..a5c9f6bff 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -573,6 +573,126 @@ PyObject* prepareBlankInstanceOfType(MutuallyRecursiveTypeGroup* outGroup, PyTyp return obj; } +/* static */ +Type* PythonSerializationContext::constructBlankInstanceOfType(Type::TypeCategory typeCat) { + if (typeCat == Type::TypeCategory::catNone) { + return NoneType::Make(); + } + if (typeCat == Type::TypeCategory::catBool) { + return Bool::Make(); + } + if (typeCat == Type::TypeCategory::catUInt8) { + return UInt8::Make(); + } + if (typeCat == Type::TypeCategory::catUInt16) { + return UInt16::Make(); + } + if (typeCat == Type::TypeCategory::catUInt32) { + return UInt32::Make(); + } + if (typeCat == Type::TypeCategory::catUInt64) { + return UInt64::Make(); + } + if (typeCat == Type::TypeCategory::catInt8) { + return Int8::Make(); + } + if (typeCat == Type::TypeCategory::catInt16) { + return Int16::Make(); + } + if (typeCat == Type::TypeCategory::catInt32) { + return Int32::Make(); + } + if (typeCat == Type::TypeCategory::catInt64) { + return Int64::Make(); + } + if (typeCat == Type::TypeCategory::catString) { + return StringType::Make(); + } + if (typeCat == Type::TypeCategory::catBytes) { + return BytesType::Make(); + } + if (typeCat == Type::TypeCategory::catFloat32) { + return Float32::Make(); + } + if (typeCat == Type::TypeCategory::catFloat64) { + return Float64::Make(); + } + if (typeCat == Type::TypeCategory::catValue) { + return new Value(); + } + if (typeCat == Type::TypeCategory::catOneOf) { + return new OneOfType(); + } + if (typeCat == Type::TypeCategory::catTupleOf) { + return new TupleOfType(); + } + if (typeCat == Type::TypeCategory::catPointerTo) { + return new PointerTo(); + } + if (typeCat == Type::TypeCategory::catListOf) { + return new ListOfType(); + } + if (typeCat == Type::TypeCategory::catNamedTuple) { + return new NamedTuple(); + } + if (typeCat == Type::TypeCategory::catTuple) { + return new Tuple(); + } + if (typeCat == Type::TypeCategory::catDict) { + return new DictType(); + } + if (typeCat == Type::TypeCategory::catConstDict) { + return new ConstDictType(); + } + if (typeCat == Type::TypeCategory::catAlternative) { + return new Alternative(); + } + if (typeCat == Type::TypeCategory::catConcreteAlternative) { + return new ConcreteAlternative(); + } + if (typeCat == Type::TypeCategory::catPythonObjectOfType) { + return new PythonObjectOfType(); + } + if (typeCat == Type::TypeCategory::catBoundMethod) { + return new BoundMethod(); + } + if (typeCat == Type::TypeCategory::catAlternativeMatcher) { + return new AlternativeMatcher(); + } + if (typeCat == Type::TypeCategory::catClass) { + return new Class(); + } + if (typeCat == Type::TypeCategory::catHeldClass) { + return new HeldClass(); + } + if (typeCat == Type::TypeCategory::catFunction) { + return new Function(); + } + if (typeCat == Type::TypeCategory::catForward) { + throw std::runtime_error("Can't deserialize actual forwards!"); + } + if (typeCat == Type::TypeCategory::catEmbeddedMessage) { + return new EmbeddedMessageType(); + } + if (typeCat == Type::TypeCategory::catSet) { + return new SetType(); + } + if (typeCat == Type::TypeCategory::catTypedCell) { + return new TypedCellType(); + } + if (typeCat == Type::TypeCategory::catPyCell) { + return PyCellType::Make(); + } + if (typeCat == Type::TypeCategory::catRefTo) { + return new RefTo(); + } + if (typeCat == Type::TypeCategory::catSubclassOf) { + return new SubclassOfType(); + } + + throw std::runtime_error("Corrupt TypeCategory: can't produce a blank instance"); +} + MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecursiveTypeGroup( DeserializationBuffer& b, size_t inWireType ) const { @@ -590,7 +710,10 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur std::map indexOfObjToIndexOfType; std::map indicesOfNativeTypes; - std::map indicesOfNativeTypeCategories; + std::set indicesOfForwardsNeedingResolution; + + std::map indicesOfNativeTypeNames; + std::map indicesWrittenAsObjectAndRep; std::map indicesWithSerializedBodies; @@ -657,21 +780,22 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur assertWireTypesEqual(subSubWireType, WireType::BYTES); std::string fwdName = b.readStringObject(); - if (actuallyBuildGroup) { - Forward* f = Forward::Make(fwdName); - - outGroup->setIndexToObject(indexInGroup, f); - indicesOfNativeTypes[indexInGroup] = f; - } else { - indicesOfNativeTypes[indexInGroup] = nullptr; - } + indicesOfNativeTypeNames[indexInGroup] = fwdName; setSomething = true; } else if (fieldInIndex == 2 && kind == 0) { assertWireTypesEqual(subSubWireType, WireType::VARINT); - indicesOfNativeTypeCategories[indexInGroup] = Type::TypeCategory( - b.readUnsignedVarint() - ); + + auto typeCat = Type::TypeCategory(b.readUnsignedVarint()); + + if (actuallyBuildGroup) { + Type* type = constructBlankInstanceOfType(typeCat); + outGroup->setIndexToObject(indexInGroup, type); + indicesOfNativeTypes[indexInGroup] = type; + } else { + indicesOfNativeTypes[indexInGroup] = nullptr; + } + } else if (fieldInIndex == 1 && kind == 1) { // this is the now-deprecated field code for sending a named object. @@ -944,12 +1068,12 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur ((Forward*)t)->getTarget() ); } else { - ((Forward*)indicesOfNativeTypes[indexInGroup])->define(t); - indicesOfNativeTypes[indexInGroup] = t; + if (t->getTypeCategory() != indicesOfNativeTypes[indexInGroup]->getTypeCategory()) { + throw std::runtime_error("Somehow, type categories didn't match!"); + } - // update the mutually recursive group or we'll end up with - // downstream consumers actually pulling out the forwards - outGroup->setIndexToObject(indexInGroup, t); + // initialize the real funtion from the function stub + indicesOfNativeTypes[indexInGroup]->initializeFrom(t); } } } else { diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index e899eb1f7..32b5f4adb 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -19,7 +19,7 @@ #include "Type.hpp" class RefTo : public Type { -protected: +public: typedef void* instance; // construct a non-forward defined refto @@ -29,7 +29,6 @@ class RefTo : public Type { m_is_default_constructible = true; } -public: RefTo(Type* t) : Type(TypeCategory::catRefTo), m_element_type(t) diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index 5ee629e0e..78fb1023c 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -27,10 +27,11 @@ PyDoc_STRVAR(Set_doc, ); class SetType : public Type { +public: SetType() : Type(TypeCategory::catSet) { } -public: + SetType(Type* eltype) : Type(TypeCategory::catSet) , m_key_type(eltype) diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index 7d52c539b..e4efa310e 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -25,12 +25,13 @@ PyDoc_STRVAR(SubclassOf_doc, ); class SubclassOfType : public Type { +public: SubclassOfType() : Type(TypeCategory::catSubclassOf) { m_is_default_constructible = true; m_size = sizeof(Type*); } -public: + SubclassOfType(Type* subclassOf) noexcept : Type(TypeCategory::catSubclassOf), m_subclassOf(subclassOf) diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index e92f5ecf9..161e43846 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -20,16 +20,6 @@ #include "Format.hpp" class TupleOrListOfType : public Type { -protected: - // this is the non-forward clone pathway - TupleOrListOfType(bool isTuple) : - Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), - m_element_type(nullptr), - m_is_tuple(isTuple) - { - m_is_default_constructible = true; - } - public: class layout { public: @@ -42,7 +32,15 @@ class TupleOrListOfType : public Type { typedef layout* layout_ptr; -public: + // this is the non-forward clone pathway + TupleOrListOfType(bool isTuple) : + Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), + m_element_type(nullptr), + m_is_tuple(isTuple) + { + m_is_default_constructible = true; + } + TupleOrListOfType(Type* type, bool isTuple) : Type(isTuple ? TypeCategory::catTupleOf : TypeCategory::catListOf), m_element_type(type), @@ -619,11 +617,11 @@ PyDoc_STRVAR(ListOf_doc, class ListOfType : public TupleOrListOfType { friend class TupleOrListOfType; +public: ListOfType() : TupleOrListOfType(false) { } -public: ListOfType(Type* type) : TupleOrListOfType(type, false) { } @@ -684,12 +682,11 @@ PyDoc_STRVAR(TupleOf_doc, class TupleOfType : public TupleOrListOfType { friend class TupleOrListOfType; - // clone form +public: TupleOfType() : TupleOrListOfType(true) { } -public: // forward form TupleOfType(Type* type) : TupleOrListOfType(type, true) { diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 2b4545937..3e8bdd00e 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -271,15 +271,6 @@ class Type { ); } - bool isUInt8() const { return m_typeCategory == catUInt8; } - bool isUInt16() const { return m_typeCategory == catUInt16; } - bool isUInt32() const { return m_typeCategory == catUInt32; } - bool isUInt64() const { return m_typeCategory == catUInt64; } - bool isInt8() const { return m_typeCategory == catInt8; } - bool isInt16() const { return m_typeCategory == catInt16; } - bool isInt32() const { return m_typeCategory == catInt32; } - bool isInt() const { return m_typeCategory == catInt64; } - bool isRecursive() { return getRecursiveTypeGroupMembers().size() != 1; } @@ -957,16 +948,6 @@ class Type { // we will have thrown, or m_forward_resolves_to will be populated. void attemptToResolve(); - // initialize ourself as a copy of 'forwardDefinitionOfSelf' where none of the types - // will have a forward definition reference. The instance must have been constructed - // using 'cloneForForwardResolution'. If the recursiveNameOverride is not None, then - // use that name for the type instead of the computed name. - void initializeFrom(Type* forwardDefinitionOfSelf) { - this->check([&](auto& subtype) { - return subtype.initializeFromConcrete(forwardDefinitionOfSelf); - }); - } - // this is a fully-resolved type that we created but aren't using. // it should just leak and never be used again. void markRedundant() { @@ -1026,6 +1007,16 @@ class Type { } public: + // initialize ourself as a copy of 'forwardDefinitionOfSelf' where none of the types + // will have a forward definition reference. The instance must have been constructed + // using 'cloneForForwardResolution'. If the recursiveNameOverride is not None, then + // use that name for the type instead of the computed name. + void initializeFrom(Type* forwardDefinitionOfSelf) { + this->check([&](auto& subtype) { + return subtype.initializeFromConcrete(forwardDefinitionOfSelf); + }); + } + // finish initializing the type assuming no forward types are reachable bool postInitialize() { size_t oldSize = m_size; diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index eabdb6ca9..678c140f1 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -21,9 +21,6 @@ //wraps a python Cell class TypedCellType : public Type { - TypedCellType() : Type(TypeCategory::catTypedCell) - { - } public: class layout { public: @@ -34,6 +31,10 @@ class TypedCellType : public Type { typedef layout* layout_ptr; + TypedCellType() : Type(TypeCategory::catTypedCell) + { + } + TypedCellType(Type* heldType) : Type(TypeCategory::catTypedCell), mHeldType(heldType) diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index 4940847fb..a65c7070f 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -24,6 +24,11 @@ PyDoc_STRVAR(Value_doc, class Value : public Type { public: + Value(const Instance& instance); + + // this is the clone-pathway + Value(); + const char* docConcrete() { return Value_doc; } @@ -118,12 +123,6 @@ class Value : public Type { const std::map& groupMap ); -private: - Value(const Instance& instance); - - // this is the clone-pathway - Value(); - Instance mInstance; PyObject* mValueAsPyobj; }; From b578cc8316fd6bb16ecd5377cb37c7ea16b8ffb9 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 8 Jun 2023 22:46:23 +0000 Subject: [PATCH 25/83] More --- typed_python/AlternativeMatcherType.hpp | 7 +- typed_python/AlternativeType.cpp | 36 +- typed_python/AlternativeType.hpp | 39 +- typed_python/BoundMethodType.hpp | 5 + typed_python/ClassType.cpp | 3 +- typed_python/ClassType.hpp | 5 + typed_python/CompositeType.hpp | 16 + typed_python/ConcreteAlternativeType.cpp | 25 -- typed_python/ConcreteAlternativeType.hpp | 8 +- typed_python/ConstDictType.hpp | 5 + typed_python/DictType.hpp | 5 + typed_python/FunctionType.hpp | 18 + typed_python/HeldClassType.hpp | 23 + typed_python/MutuallyRecursiveTypeGroup.cpp | 7 + typed_python/PointerToType.hpp | 4 + typed_python/PyAlternativeInstance.cpp | 2 +- typed_python/PyInstance.cpp | 6 +- typed_python/PythonObjectOfTypeType.cpp | 10 +- typed_python/PythonObjectOfTypeType.hpp | 50 +-- typed_python/PythonSerializationContext.hpp | 2 +- ...onSerializationContext_deserialization.cpp | 411 +++++++++--------- ...thonSerializationContext_serialization.cpp | 58 ++- typed_python/RefToType.hpp | 4 + typed_python/SetType.hpp | 4 + typed_python/SubclassOfType.hpp | 10 +- typed_python/TupleOrListOfType.hpp | 4 + typed_python/Type.cpp | 14 +- typed_python/Type.hpp | 18 +- typed_python/TypeOrPyobj.cpp | 22 +- typed_python/TypedCellType.hpp | 4 + typed_python/types_serialization_test.py | 10 +- 31 files changed, 516 insertions(+), 319 deletions(-) diff --git a/typed_python/AlternativeMatcherType.hpp b/typed_python/AlternativeMatcherType.hpp index 3f06f04b5..4472f81e6 100644 --- a/typed_python/AlternativeMatcherType.hpp +++ b/typed_python/AlternativeMatcherType.hpp @@ -34,9 +34,7 @@ class AlternativeMatcher : public Type { AlternativeMatcher(Type* inAlternative) : Type(TypeCategory::catAlternativeMatcher) { m_is_forward_defined = true; - m_is_default_constructible = false; m_alternative = inAlternative; - m_size = inAlternative->bytecount(); } const char* docConcrete() { @@ -49,6 +47,10 @@ class AlternativeMatcher : public Type { + ")"; } + void initializeDuringDeserialization(Type* altType) { + m_alternative = altType; + } + template void _visitCompilerVisibleInternals(const visitor_type& v) { v.visitHash(ShaHash(1, m_typeCategory)); @@ -66,6 +68,7 @@ class AlternativeMatcher : public Type { } void postInitializeConcrete() { + m_is_default_constructible = false; m_size = m_alternative->bytecount(); } diff --git a/typed_python/AlternativeType.cpp b/typed_python/AlternativeType.cpp index 225e0d04f..35828ca8b 100644 --- a/typed_python/AlternativeType.cpp +++ b/typed_python/AlternativeType.cpp @@ -183,8 +183,13 @@ Alternative* Alternative::Make( } } + // initialize a forward Alternative Alternative* res = new Alternative(name, moduleName, types, methods); + // construct its subtypes - we are always mutually recursive with these so we need + // to instantiate them directly in the case of the forward + res->initializeConcreteSubclasses(); + if (anyForward) { return res; } @@ -194,25 +199,34 @@ Alternative* Alternative::Make( return concrete; } -Type* Alternative::concreteSubtype(size_t which) { - if (!m_subtypes_concrete.size()) { - for (long k = 0; k < m_subtypes.size(); k++) { - m_subtypes_concrete.push_back(ConcreteAlternative::Make(this, k)); - } +void Alternative::initializeConcreteSubclasses() { + if (!m_is_forward_defined) { + throw std::runtime_error( + "Alternative::initializeConcreteSubclasses only makes sense for Forward-defined " + "Alternatives." + ); + } + + for (long k = 0; k < m_subtypes.size(); k++) { + m_subtypes_concrete.push_back(new ConcreteAlternative(this, k)); + } +} + +ConcreteAlternative* Alternative::concreteSubtype(size_t which) { + if (m_subtypes_concrete.size() != m_subtypes.size()) { + throw std::runtime_error("Corrupt Alternative - somehow we don't have a subtype."); } if (which < 0 || which >= m_subtypes_concrete.size()) { throw std::runtime_error("Invalid alternative index."); } - return m_subtypes_concrete[which]; + return (ConcreteAlternative*)m_subtypes_concrete[which]; } Type* Alternative::pickConcreteSubclassConcrete(instance_ptr data) { - if (!m_subtypes_concrete.size()) { - for (long k = 0; k < m_subtypes.size(); k++) { - m_subtypes_concrete.push_back(ConcreteAlternative::Make(this, k)); - } + if (m_subtypes_concrete.size() != m_subtypes.size()) { + throw std::runtime_error("Corrupt Alternative - somehow we don't have a subtype."); } uint8_t i = which(data); @@ -224,7 +238,7 @@ void Alternative::constructor(instance_ptr self) { assertForwardsResolvedSufficientlyToInstantiate(); if (!m_default_construction_type) { - m_default_construction_type = ConcreteAlternative::Make(this, m_default_construction_ix); + m_default_construction_type = concreteSubtype(m_default_construction_ix); } m_default_construction_type->constructor(self); diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index b3e55da26..2ec315a82 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -19,6 +19,8 @@ #include "Type.hpp" #include "CompositeType.hpp" +class ConcreteAlternative; + PyDoc_STRVAR(Alternative_doc, "Alternative(\n" " name,\n" @@ -108,7 +110,6 @@ class Alternative : public Type { m_name = name; m_moduleName = moduleName; - m_hasGetAttributeMagicMethod = m_methods.find("__getattribute__") != m_methods.end(); if (m_subtypes.size() > 255) { throw std::runtime_error("Can't have an alternative with more than 255 subelements"); @@ -119,6 +120,20 @@ class Alternative : public Type { return Alternative_doc; } + void initializeDuringDeserialization( + std::string name, + std::string moduleName, + const std::vector >& types, + const std::map& methods, + const std::vector& subtypes_concrete + ) { + m_name = name; + m_moduleName = moduleName; + m_methods = methods; + m_subtypes = types; + m_subtypes_concrete = subtypes_concrete; + } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { Alternative* selfT = (Alternative*)forwardDefinitionOfSelf; @@ -137,6 +152,8 @@ class Alternative : public Type { } void postInitializeConcrete() { + m_hasGetAttributeMagicMethod = m_methods.find("__getattribute__") != m_methods.end(); + m_arg_positions.clear(); m_default_construction_type = nullptr; @@ -216,20 +233,16 @@ class Alternative : public Type { template void _visitReferencedTypes(const visitor_type& visitor) { - for (auto& subtype_pair: m_subtypes) { - Type* t = (Type*)subtype_pair.second; - visitor(t); - assert(t == subtype_pair.second); + for (auto subtype_pair: m_subtypes) { + visitor(subtype_pair.second); } - for (long k = 0; k < m_subtypes.size(); k++) { - Type* t = concreteSubtype(k); + + for (auto t: m_subtypes_concrete) { visitor(t); } - for (auto& method_pair: m_methods) { - Type* t = (Type*)method_pair.second; - visitor(t); - assert(t == method_pair.second); + for (auto method_pair: m_methods) { + visitor(method_pair.second); } } @@ -405,9 +418,11 @@ class Alternative : public Type { return m_methods; } - Type* concreteSubtype(size_t which); + ConcreteAlternative* concreteSubtype(size_t which); private: + void initializeConcreteSubclasses(); + //name of the module in which this Alternative was defined. std::string m_moduleName; diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index 36e9d60bc..9dbf95c48 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -61,6 +61,11 @@ class BoundMethod : public Type { visitor(m_first_arg); } + void initializeDuringDeserialization(std::string funcName, Type* firstArgType) { + m_first_arg = firstArgType; + m_funcName = funcName; + } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return "BoundMethod(" + m_first_arg->computeRecursiveName(typeStack) diff --git a/typed_python/ClassType.cpp b/typed_python/ClassType.cpp index e3d996ed1..86ae4b41d 100644 --- a/typed_python/ClassType.cpp +++ b/typed_python/ClassType.cpp @@ -96,7 +96,8 @@ bool Class::cmp(instance_ptr left, instance_ptr right, int pyComparisonOp, bool pyComparisonOp == Py_GE ? ">=" : "?", name().c_str(), name().c_str() - ); + ); + throw PythonExceptionSet(); } diff --git a/typed_python/ClassType.hpp b/typed_python/ClassType.hpp index 400ce02d0..d0889e9e5 100644 --- a/typed_python/ClassType.hpp +++ b/typed_python/ClassType.hpp @@ -64,6 +64,11 @@ class Class : public Type { return Class_doc; } + void initializeDuringDeserialization(std::string inName, HeldClass* held) { + m_heldClass = held; + m_name = inName; + } + void postInitializeConcrete() { m_size = sizeof(layout*); m_is_default_constructible = m_heldClass->is_default_constructible(); diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index 9702aae51..e26acec3a 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -43,6 +43,8 @@ class CompositeType : public Type { for (long k = 0; k < names.size(); k++) { m_nameToIndex[names[k]] = k; } + + recomputeName(); } template @@ -335,6 +337,14 @@ class NamedTuple : public CompositeType { assert(types.size() == names.size()); } + void initializeDuringDeserialization( + const std::vector& types, + const std::vector& names + ) { + m_names = names; + m_types = types; + } + const char* docConcrete() { return NamedTuple_doc; } @@ -372,6 +382,12 @@ class Tuple : public CompositeType { { } + void initializeDuringDeserialization( + const std::vector& types + ) { + m_types = types; + } + const char* docConcrete() { return Tuple_doc; } diff --git a/typed_python/ConcreteAlternativeType.cpp b/typed_python/ConcreteAlternativeType.cpp index 1900a4681..8560ecb9f 100644 --- a/typed_python/ConcreteAlternativeType.cpp +++ b/typed_python/ConcreteAlternativeType.cpp @@ -28,28 +28,3 @@ void ConcreteAlternative::constructor(instance_ptr self) { }); } } - -// static -ConcreteAlternative* ConcreteAlternative::Make(Alternative* alt, int64_t which) { - if (alt->isForwardDefined()) { - return new ConcreteAlternative(alt, which); - } - - PyEnsureGilAcquired getTheGil; - - typedef std::pair keytype; - - static std::map memo; - - auto it = memo.find(keytype(alt, which)); - - if (it != memo.end()) { - return it->second; - } - - ConcreteAlternative* res = new ConcreteAlternative(alt, which); - ConcreteAlternative* concrete = (ConcreteAlternative*)res->forwardResolvesTo(); - - memo[keytype(alt, which)] = concrete; - return concrete; -} diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index 359d90fc0..dbffbbe47 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -39,6 +39,12 @@ class ConcreteAlternative : public Type { m_name = m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; } + void initializeDuringDeserialization(int64_t which, Alternative* base) { + m_alternative = base; + m_base = base; + m_which = which; + } + const char* docConcrete() { return m_alternative->doc(); } @@ -192,8 +198,6 @@ class ConcreteAlternative : public Type { return m_alternative->eltPtr(self); } - static ConcreteAlternative* Make(Alternative* alt, int64_t which); - Type* elementType() const { return m_alternative->subtypes()[m_which].second; } diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index 123d520d7..e02c8e3e4 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -52,6 +52,11 @@ class ConstDictType : public Type { m_is_forward_defined = true; } + void initializeDuringDeserialization(Type* keyType, Type* valType) { + m_key = keyType; + m_value = valType; + } + const char* docConcrete() { return ConstDictType_doc; } diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index f9cfdc107..341d53319 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -45,6 +45,11 @@ class DictType : public Type { m_is_forward_defined = true; } + void initializeDuringDeserialization(Type* keyType, Type* valType) { + m_key = keyType; + m_value = valType; + } + const char* docConcrete() { return DictType_doc; } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index e1be36a02..e9233b3d8 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1545,6 +1545,24 @@ class Function : public Type { m_name = mRootName; } + void initializeDuringDeserialization( + std::string inName, + std::string qualname, + std::string moduleName, + std::vector overloads, + Type* closureType, + bool isEntrypoint, + bool isNocompile + ) { + mRootName = inName; + mQualname = qualname; + mModulename = moduleName; + mOverloads = overloads; + mClosureType = closureType; + mIsEntrypoint = isEntrypoint; + mIsNocompile = isNocompile; + } + std::string moduleNameConcrete() { if (mModulename.size() == 0) { return "builtins"; diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 298089ccd..b3b8d6569 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -416,6 +416,29 @@ class HeldClass : public Type { } } + void initializeDuringDeserialization( + std::string inName, + const std::vector& bases, + bool isFinal, + const std::vector& members, + const std::map& memberFunctions, + const std::map& staticFunctions, + const std::map& propertyFunctions, + const std::map& classMembers, + const std::map& classMethods, + Class* clsType + ) { + m_name = inName; + m_bases = bases; + m_is_final = isFinal; + m_own_members = members; + m_own_staticFunctions = staticFunctions; + m_own_propertyFunctions = propertyFunctions; + m_own_classMembers = classMembers; + m_own_classMethods = classMethods; + m_classType = clsType; + } + const char* docConcrete() { return HeldClass_doc; } diff --git a/typed_python/MutuallyRecursiveTypeGroup.cpp b/typed_python/MutuallyRecursiveTypeGroup.cpp index 016533d37..c27fcbb6a 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.cpp +++ b/typed_python/MutuallyRecursiveTypeGroup.cpp @@ -132,6 +132,13 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { mIntendedHashToGroup[mVisibilityType][mIntendedHash] = this; } } + + // now internalize any types in here + for (auto indexAndTopo: mIndexToObject) { + if (indexAndTopo.second.type()) { + indexAndTopo.second.type()->internalize(); + } + } } void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { diff --git a/typed_python/PointerToType.hpp b/typed_python/PointerToType.hpp index 8457280ed..23ecabeff 100644 --- a/typed_python/PointerToType.hpp +++ b/typed_python/PointerToType.hpp @@ -57,6 +57,10 @@ class PointerTo : public Type { m_is_default_constructible = true; } + void initializeDuringDeserialization(Type* eltType) { + m_element_type = eltType; + } + const char* docConcrete() { return PointerTo_doc; } diff --git a/typed_python/PyAlternativeInstance.cpp b/typed_python/PyAlternativeInstance.cpp index 23e502042..d4c68d6eb 100644 --- a/typed_python/PyAlternativeInstance.cpp +++ b/typed_python/PyAlternativeInstance.cpp @@ -245,7 +245,7 @@ void PyAlternativeInstance::mirrorTypeInformationIntoPyTypeConcrete(Alternative* PyObjectStealer alternatives(PyTuple_New(alt->subtypes().size())); for (long k = 0; k < alt->subtypes().size(); k++) { - ConcreteAlternative* concrete = ConcreteAlternative::Make(alt, k); + ConcreteAlternative* concrete = alt->concreteSubtype(k); PyDict_SetItemString( pyType->tp_dict, diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 8d598e0bd..3ea72f3c8 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1727,8 +1727,7 @@ Type* PyInstance::tryUnwrapPyInstanceToType(PyObject* arg) { } return PythonObjectOfType::Make( - (PyTypeObject*)nonType, - arg + (PyTypeObject*)nonType ); } @@ -1781,8 +1780,7 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { } return PythonObjectOfType::Make( - (PyTypeObject*)nonType, - typearg + (PyTypeObject*)nonType ); } diff --git a/typed_python/PythonObjectOfTypeType.cpp b/typed_python/PythonObjectOfTypeType.cpp index c20c2d9dd..fb4cacdf5 100644 --- a/typed_python/PythonObjectOfTypeType.cpp +++ b/typed_python/PythonObjectOfTypeType.cpp @@ -506,20 +506,22 @@ bool PythonObjectOfType::cmp(instance_ptr left, instance_ptr right, int pyCompar } // static -PythonObjectOfType* PythonObjectOfType::Make(PyTypeObject* pyType, PyObject* givenType) { +PythonObjectOfType* PythonObjectOfType::Make(PyTypeObject* pyType) { PyEnsureGilAcquired getTheGil; - typedef std::pair keytype; + typedef PyTypeObject* keytype; static std::map m; - keytype key(pyType, givenType); + keytype key(pyType); auto it = m.find(key); if (it == m.end()) { + Type* t = new PythonObjectOfType(pyType); + it = m.insert( - std::make_pair(key, new PythonObjectOfType(pyType, givenType)) + std::make_pair(key, (PythonObjectOfType*)t->forwardResolvesTo()) ).first; } diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index fed88882f..92d586ffb 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -118,36 +118,23 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { { } - PythonObjectOfType(PyTypeObject* typePtr, PyObject* givenType) : + PythonObjectOfType(PyTypeObject* typePtr) : PyObjectHandleTypeBase(TypeCategory::catPythonObjectOfType) { mPyTypePtr = (PyTypeObject*)incref((PyObject*)typePtr); - - if (givenType) { - mGivenType = incref(givenType); - } else { - mGivenType = nullptr; - } - - m_name = std::string("PythonObjectOfType(") + mPyTypePtr->tp_name + ")"; - - m_size = sizeof(layout_type*); - - int isinst = PyObject_IsInstance(Py_None, (PyObject*)mPyTypePtr); - if (isinst == -1) { - isinst = 0; - PyErr_Clear(); - } - - m_is_default_constructible = isinst != 0; m_is_forward_defined = true; + recomputeName(); } void initializeFromConcrete(Type* forwardDefinitionOfSelf) { mPyTypePtr = ((PythonObjectOfType*)forwardDefinitionOfSelf)->mPyTypePtr; - mGivenType = ((PythonObjectOfType*)forwardDefinitionOfSelf)->mGivenType; m_name = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_name; m_size = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_size; + m_is_default_constructible = ((PythonObjectOfType*)forwardDefinitionOfSelf)->m_is_default_constructible; + } + + void initializeDuringDeserialization(PyTypeObject* typePtr) { + mPyTypePtr = (PyTypeObject*)incref((PyObject*)typePtr); } void updateInternalTypePointersConcrete( @@ -173,7 +160,21 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { void _visitReferencedTypes(const visitor_type& visitor) { } - void postInitializeConcrete() {} + std::string computeRecursiveNameConcrete(TypeStack& types) { + return std::string("PythonObjectOfType(") + mPyTypePtr->tp_name + ")"; + } + + void postInitializeConcrete() { + m_size = sizeof(layout_type*); + + int isinst = PyObject_IsInstance(Py_None, (PyObject*)mPyTypePtr); + if (isinst == -1) { + isinst = 0; + PyErr_Clear(); + } + + m_is_default_constructible = isinst != 0; + } int64_t refcount(instance_ptr self) const { return getHandlePtr(self)->refcount; @@ -223,7 +224,7 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { // construct a new Type. If 'givenType' is not NULL, then it's the type the user // gave us (and we inferred a real type from it based on the lookup table in internals.py) - static PythonObjectOfType* Make(PyTypeObject* pyType, PyObject* givenType=NULL); + static PythonObjectOfType* Make(PyTypeObject* pyType); static PythonObjectOfType* AnyPyObject(); @@ -235,9 +236,4 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { private: PyTypeObject* mPyTypePtr; - - // this is the object we were given, which in some cases might not really - // be a type, but which users expect to refer to a type (for instance, a threading.RLock, - // or anything else in internals._nonTypesAcceptedAsTypes); - PyObject* mGivenType; }; diff --git a/typed_python/PythonSerializationContext.hpp b/typed_python/PythonSerializationContext.hpp index 1964b0822..9578835bc 100644 --- a/typed_python/PythonSerializationContext.hpp +++ b/typed_python/PythonSerializationContext.hpp @@ -165,7 +165,7 @@ class PythonSerializationContext : public SerializationContext { static Type* constructBlankInstanceOfType(Type::TypeCategory typeCat); - Type* deserializeNativeTypeInner(DeserializationBuffer& b, size_t wireType, bool actuallyProduceResult) const; + void deserializeNativeTypeIntoBlank(DeserializationBuffer& b, size_t wireType, Type* blankShell, std::string inName) const; Instance deserializeNativeInstance(DeserializationBuffer& b, size_t wireType) const; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index a5c9f6bff..20b94b8f8 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -575,48 +575,6 @@ PyObject* prepareBlankInstanceOfType(MutuallyRecursiveTypeGroup* outGroup, PyTyp /* static */ Type* PythonSerializationContext::constructBlankInstanceOfType(Type::TypeCategory typeCat) { - if (typeCat == Type::TypeCategory::catNone) { - return NoneType::Make(); - } - if (typeCat == Type::TypeCategory::catBool) { - return Bool::Make(); - } - if (typeCat == Type::TypeCategory::catUInt8) { - return UInt8::Make(); - } - if (typeCat == Type::TypeCategory::catUInt16) { - return UInt16::Make(); - } - if (typeCat == Type::TypeCategory::catUInt32) { - return UInt32::Make(); - } - if (typeCat == Type::TypeCategory::catUInt64) { - return UInt64::Make(); - } - if (typeCat == Type::TypeCategory::catInt8) { - return Int8::Make(); - } - if (typeCat == Type::TypeCategory::catInt16) { - return Int16::Make(); - } - if (typeCat == Type::TypeCategory::catInt32) { - return Int32::Make(); - } - if (typeCat == Type::TypeCategory::catInt64) { - return Int64::Make(); - } - if (typeCat == Type::TypeCategory::catString) { - return StringType::Make(); - } - if (typeCat == Type::TypeCategory::catBytes) { - return BytesType::Make(); - } - if (typeCat == Type::TypeCategory::catFloat32) { - return Float32::Make(); - } - if (typeCat == Type::TypeCategory::catFloat64) { - return Float64::Make(); - } if (typeCat == Type::TypeCategory::catValue) { return new Value(); } @@ -671,18 +629,12 @@ Type* PythonSerializationContext::constructBlankInstanceOfType(Type::TypeCategor if (typeCat == Type::TypeCategory::catForward) { throw std::runtime_error("Can't deserialize actual forwards!"); } - if (typeCat == Type::TypeCategory::catEmbeddedMessage) { - return new EmbeddedMessageType(); - } if (typeCat == Type::TypeCategory::catSet) { return new SetType(); } if (typeCat == Type::TypeCategory::catTypedCell) { return new TypedCellType(); } - if (typeCat == Type::TypeCategory::catPyCell) { - return PyCellType::Make(); - } if (typeCat == Type::TypeCategory::catRefTo) { return new RefTo(); } @@ -690,6 +642,28 @@ Type* PythonSerializationContext::constructBlankInstanceOfType(Type::TypeCategor return new SubclassOfType(); } + + + if (typeCat == Type::TypeCategory::catPyCell + || typeCat == Type::TypeCategory::catEmbeddedMessage + || typeCat == Type::TypeCategory::catNone + || typeCat == Type::TypeCategory::catBool + || typeCat == Type::TypeCategory::catUInt8 + || typeCat == Type::TypeCategory::catUInt16 + || typeCat == Type::TypeCategory::catUInt32 + || typeCat == Type::TypeCategory::catUInt64 + || typeCat == Type::TypeCategory::catInt8 + || typeCat == Type::TypeCategory::catInt16 + || typeCat == Type::TypeCategory::catInt32 + || typeCat == Type::TypeCategory::catInt64 + || typeCat == Type::TypeCategory::catString + || typeCat == Type::TypeCategory::catBytes + || typeCat == Type::TypeCategory::catFloat32 + || typeCat == Type::TypeCategory::catFloat64 + ) { + throw std::runtime_error("Can't produce a blank instance of a simple type"); + } + throw std::runtime_error("Corrupt TypeCategory: can't produce a blank instance"); } @@ -710,7 +684,6 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur std::map indexOfObjToIndexOfType; std::map indicesOfNativeTypes; - std::set indicesOfForwardsNeedingResolution; std::map indicesOfNativeTypeNames; @@ -1046,36 +1019,22 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } } else if (indicesOfNativeTypes.find(indexInGroup) != indicesOfNativeTypes.end()) { - Type* t = deserializeNativeTypeInner(b, subWireType, actuallyBuildGroup); + Type* targetType = nullptr; if (actuallyBuildGroup) { - if (!t) { - throw std::runtime_error("Somehow, deserializeNativeTypeInner didn't return a type."); - } - if (!indicesOfNativeTypes[indexInGroup]) { throw std::runtime_error("indicesOfNativeTypes[indexInGroup] is somehow none"); } - if (!indicesOfNativeTypes[indexInGroup]->isForward()) { - throw std::runtime_error("indicesOfNativeTypes[indexInGroup] was already resolved"); - } - - if (t->isForward()) { - // we actually deserialized a forward here! this means everyone - // else is expecting this actual instance to be a forward, so we - // don't want to swap it out - ((Forward*)indicesOfNativeTypes[indexInGroup])->define( - ((Forward*)t)->getTarget() - ); - } else { - if (t->getTypeCategory() != indicesOfNativeTypes[indexInGroup]->getTypeCategory()) { - throw std::runtime_error("Somehow, type categories didn't match!"); - } - // initialize the real funtion from the function stub - indicesOfNativeTypes[indexInGroup]->initializeFrom(t); - } + targetType = indicesOfNativeTypes[indexInGroup]; } + + deserializeNativeTypeIntoBlank( + b, + subWireType, + targetType, + indicesOfNativeTypeNames[indexInGroup] + ); } else { throw std::runtime_error("Corrupt MutuallyRecursiveTypeGroup group"); } @@ -1225,6 +1184,35 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } if (actuallyBuildGroup) { + // recompute type names + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->recomputeName(); + } + + // do post-initialization + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto ixAndType: indicesOfNativeTypes) { + if (ixAndType.second->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > indicesOfNativeTypes.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } + } + + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->finalizeType(); + } + outGroup->finalizeDeserializerGroup(); } @@ -1525,16 +1513,17 @@ Instance PythonSerializationContext::deserializeNativeInstance(DeserializationBu return result; } -// if actuallyProduceResult == false, then we still need to walk over the objects to make sure -// that our memos are correctly populated, but we don't need to construct objects, which will prevent us -// from creating new objects and leaking them when deserializing classes we already hold. -Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuffer& b, size_t inWireType, bool actuallyProduceResult) const { +/* deserialize a native type of the apropriate category into an empty Type* that's been constructed */ +void PythonSerializationContext::deserializeNativeTypeIntoBlank( + DeserializationBuffer& b, + size_t inWireType, + Type* blankShell, + std::string inName +) const { PyEnsureGilAcquired acquireTheGil; int category = -1; - Type* namedType = nullptr; - std::vector types; std::vector names; int64_t whichIndex = -1; @@ -1559,23 +1548,7 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff category = b.readUnsignedVarint(); } else if (wireType == WireType::BYTES) { - std::string objectName = b.readStringObject(); - - PyObjectStealer contextAsPyObj(PyInstance::extractPythonObject(mContextObj)); - - PyObjectStealer namedObj(PyObject_CallMethod(contextAsPyObj, "objectFromName", "s", objectName.c_str())); - - if (!namedObj) { - throw PythonExceptionSet(); - } - - if (!PyType_Check(namedObj)) { - throw std::runtime_error("Corrupt native type found: object named " + objectName + " was not a type."); - } - namedType = PyInstance::extractTypeFrom((PyTypeObject*)(PyObject*)namedObj); - if (!namedType) { - throw std::runtime_error("Corrupt native type found: object named " + objectName + " was not a native type."); - } + throw std::runtime_error("We shouldn't be using named serialization in an MRTG"); } else { throw std::runtime_error("Corrupt native type found: 0 field should be a category or a name"); } @@ -1608,6 +1581,13 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff deserializeAlternativeMembers(alternativeMembers, b, wireType); } else if (fieldNumber == 4) { deserializeClassFunDict(names.back(), classMethods, b, wireType); + } else if (fieldNumber == 5) { + b.consumeCompoundMessage(wireType, [&](int which, size_t wireType2) { + types.push_back(deserializeNativeType(b, wireType2)); + if (!types.back()->isConcreteAlternative()) { + throw std::runtime_error("Expected a concrete alternative"); + } + }); } } else if (category == Type::TypeCategory::catHeldClass) { @@ -1644,6 +1624,8 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff } else if (fieldNumber == 8) { assertWireTypesEqual(wireType, WireType::VARINT); classIsFinal = b.readUnsignedVarint(); + } else if (fieldNumber == 9) { + types.push_back(deserializeNativeType(b, wireType)); } } else if (category == Type::TypeCategory::catFunction) { @@ -1702,48 +1684,52 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff } }); - // if the type was just named in the codebase, return it - if (namedType) { - return namedType; - } - - if (!actuallyProduceResult) { - return nullptr; + if (!blankShell) { + // we don't actually need to construct this thing + return; } - // otherwise, construct it from the pieces we were given. - Type* resultType = nullptr; - if (category == Type::TypeCategory::catClass) { if (types.size() != 1) { throw std::runtime_error("Corrupt 'Class' encountered: can't have multiple corresponding HeldClass objects."); } - if (types[0]->getTypeCategory() != Type::TypeCategory::catHeldClass) { + if (!types[0]->isHeldClass()) { throw std::runtime_error( - "Corrupt 'Class' encountered: HeldClass is a " + - Type::categoryToString(types[0]->getTypeCategory()) + " not a HeldClass" + "Corrupt 'Class' encountered: HeldClass is a " + types[0]->getTypeCategoryString() + + " not a HeldClass" ); } - resultType = ((HeldClass*)types[0])->getClassType(); + if (!blankShell->isClass()) { + throw std::runtime_error("Shell is not a Class"); + } + + ((Class*)blankShell)->initializeDuringDeserialization(inName, (HeldClass*)types[0]); } else if (category == Type::TypeCategory::catAlternative) { if (names.size() != 2) { throw std::runtime_error("Corrupt 'Alternative' encountered: invalid number of names"); } - resultType = Alternative::Make( + if (!blankShell->isAlternative()) { + throw std::runtime_error("Shell is not an Alternative"); + } + + ((Alternative*)blankShell)->initializeDuringDeserialization( names[0], names[1], alternativeMembers, - classMethods + classMethods, + types ); } else if (category == Type::TypeCategory::catForward) { if (names.size() != 1 || types.size() != 1) { throw std::runtime_error("Corrupt Forward"); } - resultType = Forward::Make(names[0]); - ((Forward*)resultType)->define(types[0]); + if (!blankShell->isForward()) { + throw std::runtime_error("Shell is not a Forward"); + } + ((Forward*)blankShell)->define(types[0]); } else if (category == Type::TypeCategory::catHeldClass) { if (names.size() != 1) { @@ -1755,24 +1741,45 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff classClassMembersRaw[nameAndObj.first] = incref((PyObject*)nameAndObj.second); } - resultType = Class::Make( + if (!blankShell->isForward()) { + throw std::runtime_error("Shell is not a HeldClass"); + } + + if (types.size() != 1 || !types[0]->isClass()) { + throw std::runtime_error("Corrupt 'HeldClass' - needed a Class"); + } + + std::vector heldBases; + for (auto b: classBases) { + if (!b->isHeldClass()) { + throw std::runtime_error("Non-held base encountered!"); + } + heldBases.push_back((HeldClass*)b); + } + + ((HeldClass*)blankShell)->initializeDuringDeserialization( names[0], - classBases, + heldBases, classIsFinal, classMembers, classMethods, classStatics, classPropertyFunctions, classClassMembersRaw, - classClassMethods - )->getHeldClass(); + classClassMethods, + (Class*)types[0] + ); } else if (category == Type::TypeCategory::catFunction) { if (names.size() != 3) { throw std::runtime_error("Badly structured 'Function' encountered."); } - resultType = Function::Make( + if (!blankShell->isForward()) { + throw std::runtime_error("Shell is not a Function"); + } + + ((Function*)blankShell)->initializeDuringDeserialization( names[0], names[1], names[2], @@ -1782,83 +1789,50 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff isNocompile ); } - else if (category == Type::TypeCategory::catUInt8) { - resultType = ::UInt8::Make(); - } - else if (category == Type::TypeCategory::catUInt16) { - resultType = ::UInt16::Make(); - } - else if (category == Type::TypeCategory::catUInt32) { - resultType = ::UInt32::Make(); - } - else if (category == Type::TypeCategory::catUInt64) { - resultType = ::UInt64::Make(); - } - else if (category == Type::TypeCategory::catInt8) { - resultType = ::Int8::Make(); - } - else if (category == Type::TypeCategory::catInt16) { - resultType = ::Int16::Make(); - } - else if (category == Type::TypeCategory::catInt32) { - resultType = ::Int32::Make(); - } - else if (category == Type::TypeCategory::catInt64) { - resultType = ::Int64::Make(); - } - else if (category == Type::TypeCategory::catEmbeddedMessage) { - resultType = ::EmbeddedMessageType::Make(); - } - else if (category == Type::TypeCategory::catFloat32) { - resultType = ::Float32::Make(); - } - else if (category == Type::TypeCategory::catFloat64) { - resultType = ::Float64::Make(); - } - else if (category == Type::TypeCategory::catNone) { - resultType = ::NoneType::Make(); - } - else if (category == Type::TypeCategory::catBytes) { - resultType = ::BytesType::Make(); - } - else if (category == Type::TypeCategory::catString) { - resultType = ::StringType::Make(); - } - else if (category == Type::TypeCategory::catPyCell) { - resultType = ::PyCellType::Make(); - } - else if (category == Type::TypeCategory::catBool) { - resultType = ::Bool::Make(); - } - else if (category == Type::TypeCategory::catOneOf) { - resultType = ::OneOfType::Make(types); - } - else if (category == Type::TypeCategory::catValue) { - resultType = ::Value::Make(instance); - } else if (category == Type::TypeCategory::catTupleOf) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: TupleOf needs exactly 1 type."); } - resultType = ::TupleOfType::Make(types[0]); + + if (!blankShell->isTupleOf()) { + throw std::runtime_error("Shell is not a TupleOf"); + } + + ((TupleOfType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catListOf) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: ListOf needs exactly 1 type."); } - resultType = ::ListOfType::Make(types[0]); + + if (!blankShell->isListOf()) { + throw std::runtime_error("Shell is not a ListOf"); + } + + ((ListOfType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catTypedCell) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: TypedCell needs exactly 1 type."); } - resultType = ::TypedCellType::Make(types[0]); + + if (!blankShell->isTypedCell()) { + throw std::runtime_error("Shell is not a TypedCell"); + } + + ((TypedCellType*)blankShell)->initializeDuringDeserialization(types[0]); + } else if (category == Type::TypeCategory::catPointerTo) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: PointerTo needs exactly 1 type."); } - resultType = ::PointerTo::Make(types[0]); + + if (!blankShell->isPointerTo()) { + throw std::runtime_error("Shell is not a PointerTo"); + } + + ((PointerTo*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catBoundMethod) { if (types.size() != 1) { @@ -1867,56 +1841,105 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff if (names.size() != 1) { throw std::runtime_error("Invalid native type: BoundMethod needs exactly 1 name."); } - resultType = ::BoundMethod::Make(types[0], names[0]); + + if (!blankShell->isBoundMethod()) { + throw std::runtime_error("Shell is not a BoundMethod"); + } + + ((BoundMethod*)blankShell)->initializeDuringDeserialization(names[0], types[0]); } else if (category == Type::TypeCategory::catAlternativeMatcher) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: AternativeMatcher needs exactly 1 type."); } - resultType = ::AlternativeMatcher::Make(types[0]); + + if (!blankShell->isAlternativeMatcher()) { + throw std::runtime_error("Shell is not a AlternativeMatcher"); + } + + ((AlternativeMatcher*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catSubclassOf) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: SubclassOf needs exactly 1 type."); } - resultType = ::SubclassOfType::Make(types[0]); + + if (!blankShell->isSubclassOf()) { + throw std::runtime_error("Shell is not a SubclassOf"); + } + + ((SubclassOfType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catRefTo) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: RefTo needs exactly 1 type."); } - resultType = ::RefTo::Make(types[0]); + + if (!blankShell->isRefTo()) { + throw std::runtime_error("Shell is not a RefTo"); + } + + ((RefTo*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catNamedTuple) { - resultType = ::NamedTuple::Make(types, names); + if (!blankShell->isNamedTuple()) { + throw std::runtime_error("Shell is not a NamedTuple"); + } + + ((NamedTuple*)blankShell)->initializeDuringDeserialization(types, names); } else if (category == Type::TypeCategory::catTuple) { - resultType = ::Tuple::Make(types); + if (!blankShell->isTuple()) { + throw std::runtime_error("Shell is not a Tuple"); + } + + ((Tuple*)blankShell)->initializeDuringDeserialization(types); } else if (category == Type::TypeCategory::catSet) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: Set needs exactly 1 type."); } - resultType = ::SetType::Make(types[0]); + + if (!blankShell->isSet()) { + throw std::runtime_error("Shell is not a Set"); + } + + ((SetType*)blankShell)->initializeDuringDeserialization(types[0]); } else if (category == Type::TypeCategory::catDict) { if (types.size() != 2) { throw std::runtime_error("Invalid native type: Dict needs exactly 2 types."); } - resultType = ::DictType::Make(types[0], types[1]); + + if (!blankShell->isDict()) { + throw std::runtime_error("Shell is not a Dict"); + } + + ((DictType*)blankShell)->initializeDuringDeserialization(types[0], types[1]); } else if (category == Type::TypeCategory::catConstDict) { if (types.size() != 2) { throw std::runtime_error("Invalid native type: ConstDict needs exactly 2 types."); } - resultType = ::ConstDictType::Make(types[0], types[1]); + + if (!blankShell->isConstDict()) { + throw std::runtime_error("Shell is not a ConstDict"); + } + + ((ConstDictType*)blankShell)->initializeDuringDeserialization(types[0], types[1]); } else if (category == Type::TypeCategory::catPythonObjectOfType) { if (!obj || !PyType_Check(obj)) { throw std::runtime_error("Invalid native type: PythonObjectOfType needs a python type."); } - resultType = ::PythonObjectOfType::Make((PyTypeObject*)(PyObject*)obj); + if (!blankShell->isPythonObjectOfType()) { + throw std::runtime_error("Shell is not a PythonObjectOfType"); + } + + ((PythonObjectOfType*)blankShell)->initializeDuringDeserialization( + (PyTypeObject*)(PyObject*)obj + ); } else if (category == Type::TypeCategory::catConcreteAlternative) { if (types.size() != 1) { @@ -1936,16 +1959,16 @@ Type* PythonSerializationContext::deserializeNativeTypeInner(DeserializationBuff throw std::runtime_error("corrupt data: invalid alternative specified"); } - resultType = a->concreteSubtype(whichIndex); - } else { - throw std::runtime_error("Invalid native type category"); - } + if (!blankShell->isConcreteAlternative()) { + throw std::runtime_error("Shell is not a ConcreteAlternative"); + } - if (!resultType) { - throw std::runtime_error("Corrupt nativeType."); + ((ConcreteAlternative*)blankShell)->initializeDuringDeserialization(whichIndex, a); + } else { + throw std::runtime_error( + "Cant initialize a blank instance of " + blankShell->getTypeCategoryString() + ); } - - return resultType; } template diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index b8d578b3b..969b610f2 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -346,6 +346,9 @@ bool isSimpleType(Type* t) { void PythonSerializationContext::serializeNativeType(Type* nativeType, SerializationBuffer& b, size_t fieldNumber) const { PyEnsureGilAcquired getTheGil; + if (nativeType->isForwardDefined() || nativeType->isForward()) { + throw std::runtime_error("Can't serialize Forward or forward-defined types yet"); + } # if TP_VERBOSE_SERIALIZE ScopedIndenter s; @@ -362,7 +365,7 @@ void PythonSerializationContext::serializeNativeType(Type* nativeType, Serializa b.writeUnsignedVarintObject(0, 0); b.writeUnsignedVarintObject(1, nativeType->getTypeCategory()); } else - if (nativeType->getTypeCategory() == Type::TypeCategory::catConcreteAlternative && + if (nativeType->isConcreteAlternative() && (name = getNameForPyObj((PyObject*)PyInstance::typeObj(nativeType->getBaseType()))).size()) { // this is an inline named concrete alternative b.writeUnsignedVarintObject(0, 1); @@ -861,60 +864,67 @@ void PythonSerializationContext::serializeNativeTypeInner( b.writeUnsignedVarintObject(0, nativeType->getTypeCategory()); - if (nativeType->getTypeCategory() == Type::TypeCategory::catConcreteAlternative) { + if (nativeType->isConcreteAlternative()) { serializeNativeType(nativeType->getBaseType(), b, 1); b.writeUnsignedVarintObject(2, ((ConcreteAlternative*)nativeType)->which()); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catAlternative) { + } else if (nativeType->isAlternative()) { Alternative* altType = (Alternative*)nativeType; b.writeStringObject(1, altType->name()); b.writeStringObject(2, altType->moduleName()); serializeAlternativeMembers(altType->subtypes(), b, 3); serializeClassFunDict(altType->getMethods(), b, 4); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catSet) { + + b.writeBeginCompound(5); + for (long k = 0; k < altType->subtypes().size(); k++) { + serializeNativeType(altType->concreteSubtype(k), b, 1); + } + b.writeEndCompound(); + + } else if (nativeType->isSet()) { serializeNativeType(((SetType*)nativeType)->keyType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catConstDict) { + } else if (nativeType->isConstDict()) { serializeNativeType(((ConstDictType*)nativeType)->keyType(), b, 1); serializeNativeType(((ConstDictType*)nativeType)->valueType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catDict) { + } else if (nativeType->isDict()) { serializeNativeType(((DictType*)nativeType)->keyType(), b, 1); serializeNativeType(((DictType*)nativeType)->valueType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTupleOf) { + } else if (nativeType->isTupleOf()) { serializeNativeType(((TupleOfType*)nativeType)->getEltType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPointerTo) { + } else if (nativeType->isPointerTo()) { serializeNativeType(((PointerTo*)nativeType)->getEltType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catRefTo) { + } else if (nativeType->isRefTo()) { serializeNativeType(((RefTo*)nativeType)->getEltType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catSubclassOf) { + } else if (nativeType->isSubclassOf()) { serializeNativeType(((SubclassOfType*)nativeType)->getSubclassOf(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catBoundMethod) { + } else if (nativeType->isBoundMethod()) { b.writeStringObject(1, ((BoundMethod*)nativeType)->getFuncName()); serializeNativeType(((BoundMethod*)nativeType)->getFirstArgType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catAlternativeMatcher) { + } else if (nativeType->isAlternativeMatcher()) { serializeNativeType(((AlternativeMatcher*)nativeType)->getAlternative(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catListOf) { + } else if (nativeType->isListOf()) { serializeNativeType(((ListOfType*)nativeType)->getEltType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTypedCell) { + } else if (nativeType->isTypedCell()) { serializeNativeType(((TypedCellType*)nativeType)->getHeldType(), b, 2); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catTuple) { + } else if (nativeType->isTuple()) { size_t index = 1; for (auto t: ((CompositeType*)nativeType)->getTypes()) { serializeNativeType(t, b, index++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catNamedTuple) { + } else if (nativeType->isNamedTuple()) { size_t index = 1; for (long k = 0; k < ((CompositeType*)nativeType)->getTypes().size(); k++) { b.writeStringObject(index++, ((CompositeType*)nativeType)->getNames()[k]); serializeNativeType(((CompositeType*)nativeType)->getTypes()[k], b, index++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catOneOf) { + } else if (nativeType->isOneOf()) { size_t index = 1; for (auto t: ((OneOfType*)nativeType)->getTypes()) { serializeNativeType(t, b, index++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catPythonObjectOfType) { + } else if (nativeType->isPythonObjectOfType()) { serializePythonObject((PyObject*)((PythonObjectOfType*)nativeType)->pyType(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catValue) { + } else if (nativeType->isValue()) { b.writeBeginCompound(1); Instance i = ((Value*)nativeType)->value(); @@ -923,7 +933,7 @@ void PythonSerializationContext::serializeNativeTypeInner( PyEnsureGilReleased releaseTheGil; i.type()->serialize(i.data(), b, 1); b.writeEndCompound(); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catFunction) { + } else if (nativeType->isFunction()) { Function* ftype = (Function*)nativeType; serializeNativeType(ftype->getClosureType(), b, 1); @@ -944,15 +954,15 @@ void PythonSerializationContext::serializeNativeTypeInner( for (auto& overload: ftype->getOverloads()) { overload.serialize(*this, b, whichIndex++); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catForward) { + } else if (nativeType->isForward()) { b.writeStringObject(1, nativeType->name()); if (((Forward*)nativeType)->getTarget()) { serializeNativeType(((Forward*)nativeType)->getTarget(), b, 2); } - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catClass) { + } else if (nativeType->isClass()) { serializeNativeType(((Class*)nativeType)->getHeldClass(), b, 1); - } else if (nativeType->getTypeCategory() == Type::TypeCategory::catHeldClass) { + } else if (nativeType->isHeldClass()) { HeldClass* cls = (HeldClass*)nativeType; // note that we serialize the name of the class (not the held class) @@ -983,6 +993,8 @@ void PythonSerializationContext::serializeNativeTypeInner( b.writeEndCompound(); b.writeUnsignedVarintObject(8, cls->isFinal() ? 1 : 0); + + serializeNativeType(cls->getClassType(), b, 9); } else { throw std::runtime_error( "Can't serialize native type " + nativeType->name() diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index 32b5f4adb..35a8cd34e 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -65,6 +65,10 @@ class RefTo : public Type { return concrete; } + void initializeDuringDeserialization(Type* eltType) { + m_element_type = eltType; + } + bool isPODConcrete() { return true; } diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index 78fb1023c..5e53e1a4d 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -43,6 +43,10 @@ class SetType : public Type { return Set_doc; } + void initializeDuringDeserialization(Type* keyType) { + m_key_type = keyType; + } + template void _visitReferencedTypes(const visitor_type& visitor) { visitor(m_key_type); diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index e4efa310e..917f0e05e 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -28,8 +28,6 @@ class SubclassOfType : public Type { public: SubclassOfType() : Type(TypeCategory::catSubclassOf) { - m_is_default_constructible = true; - m_size = sizeof(Type*); } SubclassOfType(Type* subclassOf) noexcept : @@ -37,15 +35,19 @@ class SubclassOfType : public Type { m_subclassOf(subclassOf) { m_is_forward_defined = true; - m_is_default_constructible = true; - m_size = sizeof(Type*); } const char* docConcrete() { return SubclassOf_doc; } + void initializeDuringDeserialization(Type* subclassOf) { + m_subclassOf = subclassOf; + } + void postInitializeConcrete() { + m_is_default_constructible = true; + m_size = sizeof(Type*); } void initializeFromConcrete(Type* forwardDefinitionOfSelf) { diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 161e43846..63c717863 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -50,6 +50,10 @@ class TupleOrListOfType : public Type { m_is_default_constructible = true; } + void initializeDuringDeserialization(Type* elementType) { + m_element_type = elementType; + } + template void _visitContainedTypes(const visitor_type& visitor) { diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index ee57cda1f..8bf82b21a 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -349,7 +349,7 @@ void Type::attemptToResolve() { // step. Otherwise, it's possible for the names to depend on the order of initialization // and we want them to be stable. for (auto typeAndSource: resolutionSource) { - typeAndSource.first->recomputeNamePostInitialization(); + typeAndSource.first->recomputeName(); } // cause each type to post-initialize itself, which lets it update internal bytecounts and @@ -438,6 +438,18 @@ void Type::attemptToResolve() { } } +void Type::internalize() { + if (mIdentityHash == ShaHash()) { + throw std::runtime_error("This type should already have been hashed!"); + } + + if (mInternalizedIdentityHashToType.find(mIdentityHash) + != mInternalizedIdentityHashToType.end()) { + throw std::runtime_error("This type is already internalized!"); + } + + mInternalizedIdentityHashToType[mIdentityHash] = this; +} PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { int num_args = 0; diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 3e8bdd00e..283caef4f 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -176,10 +176,22 @@ class Type { return m_typeCategory == catSet; } + bool isPointerTo() const { + return m_typeCategory == catPointerTo; + } + bool isConstDict() const { return m_typeCategory == catConstDict; } + bool isAlternativeMatcher() const { + return m_typeCategory == catAlternativeMatcher; + } + + bool isBoundMethod() const { + return m_typeCategory == catBoundMethod; + } + bool isNamedTuple() const { return m_typeCategory == catNamedTuple; } @@ -1000,13 +1012,15 @@ class Type { static std::map mInternalizedIdentityHashToType; - void recomputeNamePostInitialization() { +public: + void recomputeName() { TypeStack stack; m_name = computeRecursiveName(stack); m_stripped_name = ""; } -public: + void internalize(); + // initialize ourself as a copy of 'forwardDefinitionOfSelf' where none of the types // will have a forward definition reference. The instance must have been constructed // using 'cloneForForwardResolution'. If the recursiveNameOverride is not None, then diff --git a/typed_python/TypeOrPyobj.cpp b/typed_python/TypeOrPyobj.cpp index 17ae4aa7a..3f82c843c 100644 --- a/typed_python/TypeOrPyobj.cpp +++ b/typed_python/TypeOrPyobj.cpp @@ -22,9 +22,13 @@ TypeOrPyobj::TypeOrPyobj(Type* t) : mPyObj(nullptr) { if (!mType) { - asm("int3"); throw std::runtime_error("Can't construct a TypeOrPyobj with a null Type"); } + + if (mType->isForward() || mType->isForwardDefined()) { + asm("int3"); + throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); + } } TypeOrPyobj::TypeOrPyobj(PyObject* o) : @@ -35,7 +39,13 @@ TypeOrPyobj::TypeOrPyobj(PyObject* o) : if (PyType_Check(mPyObj)) { mType = PyInstance::extractTypeFrom((PyTypeObject*)mPyObj, true); + if (mType) { + if (mType->isForward() || mType->isForwardDefined()) { + asm("int3"); + throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); + } + mPyObj = nullptr; return; } @@ -59,6 +69,11 @@ TypeOrPyobj::TypeOrPyobj(PyTypeObject* o) : mType = PyInstance::extractTypeFrom(o, true); if (mType) { + if (mType->isForward() || mType->isForwardDefined()) { + asm("int3"); + throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); + } + mPyObj = nullptr; return; } @@ -81,6 +96,11 @@ TypeOrPyobj TypeOrPyobj::withoutIntern(PyObject* o) { if (PyType_Check(o)) { obj.mType = PyInstance::extractTypeFrom((PyTypeObject*)o, true); if (obj.mType) { + if (obj.mType->isForward() || obj.mType->isForwardDefined()) { + asm("int3"); + throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); + } + obj.mPyObj = nullptr; return obj; } diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index 678c140f1..f39bc5180 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -50,6 +50,10 @@ class TypedCellType : public Type { return "TypedCell(" + mHeldType->computeRecursiveName(typeStack) + ")"; } + void initializeDuringDeserialization(Type* heldType) { + mHeldType = heldType; + } + void initializeFromConcrete(Type* forwardDefinitionOfSelf) { mHeldType = ((TypedCellType*)forwardDefinitionOfSelf)->mHeldType; } diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 69150abf9..9cf6bd052 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -45,7 +45,7 @@ Dict, Set, SerializationContext, EmbeddedMessage, serializeStream, deserializeStream, decodeSerializedObject, Forward, Final, Function, Entrypoint, TypeFunction, PointerTo, - SubclassOf, NotCompiled + SubclassOf, NotCompiled, resolveForwardDefinedType ) from typed_python._types import ( @@ -560,11 +560,13 @@ def f(): ts = SerializationContext({'f': f}) self.assertIs(ping_pong(f, ts), f) - def test_serialize_alternatives_as_types(self): + def test_serialize_recursive_alternative(self): A = Forward("A") - A = A.define(Alternative("A", X={'a': int}, Y={'a': A})) + A.define(Alternative("A", X={'a': int}, Y={'a': A})) + # A = resolveForwardDefinedType(A) - ts = SerializationContext({'A': A}) + + ts = SerializationContext() self.assertIs(ping_pong(A, ts), A) self.assertIs(ping_pong(A.X, ts), A.X) From 3ec6e2c9ff190fc245ae75a5389f9c99deb02303 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 9 Jun 2023 18:31:49 -0400 Subject: [PATCH 26/83] More --- typed_python/ForwardType.hpp | 2 +- typed_python/PointerToType.hpp | 10 +- typed_python/PyForwardInstance.hpp | 24 ++++ ...onSerializationContext_deserialization.cpp | 2 +- typed_python/RefToType.hpp | 12 +- typed_python/TypeOrPyobj.cpp | 20 --- typed_python/TypedCellType.hpp | 14 ++- typed_python/__init__.py | 39 +++--- typed_python/python_ast.py | 11 +- .../test_serialize_recursive_types.py | 115 ++++++++++++++++++ typed_python/test_util.py | 13 +- typed_python/type_construction_test.py | 62 ++++++++++ typed_python/type_filter.py | 42 ------- typed_python/type_identity_test.py | 5 - typed_python/types_test.py | 20 +-- 15 files changed, 267 insertions(+), 124 deletions(-) create mode 100644 typed_python/test_serialize_recursive_types.py delete mode 100644 typed_python/type_filter.py diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 34b8181a2..07d6187a5 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -23,7 +23,7 @@ PyDoc_STRVAR(Forward_doc, "Forward(n) -> new forward type named n\n" "\n" "Forward types must be resolved before any types that contain them can be used.\n" - ); +); // forward types must be resolved (removed from the graph) before // any types that contain them can be used. diff --git a/typed_python/PointerToType.hpp b/typed_python/PointerToType.hpp index 23ecabeff..8c2564df4 100644 --- a/typed_python/PointerToType.hpp +++ b/typed_python/PointerToType.hpp @@ -44,8 +44,6 @@ class PointerTo : public Type { // construct a non-forward defined pointer PointerTo() : Type(TypeCategory::catPointerTo) { - m_size = sizeof(instance); - m_is_default_constructible = true; } PointerTo(Type* t) : @@ -53,8 +51,7 @@ class PointerTo : public Type { m_element_type(t) { m_is_forward_defined = true; - m_size = sizeof(instance); - m_is_default_constructible = true; + recomputeName(); } void initializeDuringDeserialization(Type* eltType) { @@ -115,7 +112,10 @@ class PointerTo : public Type { return new PointerTo(); } - void postInitializeConcrete() {} + void postInitializeConcrete() { + m_size = sizeof(instance); + m_is_default_constructible = true; + } template void _visitContainedTypes(const visitor_type& visitor) { diff --git a/typed_python/PyForwardInstance.hpp b/typed_python/PyForwardInstance.hpp index 2c029475c..7ac26045d 100644 --- a/typed_python/PyForwardInstance.hpp +++ b/typed_python/PyForwardInstance.hpp @@ -81,10 +81,34 @@ class PyForwardInstance : public PyInstance { return incref(PyInstance::typePtrToPyTypeRepresentation(result)); } + static PyObject* forwardResolve(PyObject* o, PyObject* args) { + if (PyTuple_Size(args) != 0) { + PyErr_SetString(PyExc_TypeError, "Forward.get takes zero arguments"); + return NULL; + } + + Forward* self_type = (Forward*)PyInstance::unwrapTypeArgToTypePtr(o); + if (!self_type) { + PyErr_SetString(PyExc_TypeError, "Forward.get unexpected error"); + return NULL; + } + + return translateExceptionToPyObject([&]() { + Type* result = self_type->getTarget()->forwardResolvesTo(); + + if (!result) { + return incref(Py_None); + } + + return incref(PyInstance::typePtrToPyTypeRepresentation(result)); + }); + } + static PyMethodDef* typeMethodsConcrete(Type* t) { return new PyMethodDef [4] { {"define", (PyCFunction)PyForwardInstance::forwardDefine, METH_VARARGS | METH_CLASS, NULL}, {"get", (PyCFunction)PyForwardInstance::forwardGet, METH_VARARGS | METH_CLASS, NULL}, + {"resolve", (PyCFunction)PyForwardInstance::forwardResolve, METH_VARARGS | METH_CLASS, NULL}, {NULL, NULL} }; } diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 20b94b8f8..8d095d64f 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1775,7 +1775,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( throw std::runtime_error("Badly structured 'Function' encountered."); } - if (!blankShell->isForward()) { + if (!blankShell->isFunction()) { throw std::runtime_error("Shell is not a Function"); } diff --git a/typed_python/RefToType.hpp b/typed_python/RefToType.hpp index 35a8cd34e..2442e266d 100644 --- a/typed_python/RefToType.hpp +++ b/typed_python/RefToType.hpp @@ -33,9 +33,8 @@ class RefTo : public Type { Type(TypeCategory::catRefTo), m_element_type(t) { - m_size = sizeof(instance); - m_is_default_constructible = false; m_is_forward_defined = true; + recomputeName(); } template @@ -65,6 +64,10 @@ class RefTo : public Type { return concrete; } + const char* docConcrete() { + return "RefTo is deprecated."; + } + void initializeDuringDeserialization(Type* eltType) { m_element_type = eltType; } @@ -89,7 +92,10 @@ class RefTo : public Type { return new RefTo(); } - void postInitializeConcrete() {} + void postInitializeConcrete() { + m_size = sizeof(instance); + m_is_default_constructible = false; + } template void _visitContainedTypes(const visitor_type& visitor) { diff --git a/typed_python/TypeOrPyobj.cpp b/typed_python/TypeOrPyobj.cpp index 3f82c843c..226f3ef93 100644 --- a/typed_python/TypeOrPyobj.cpp +++ b/typed_python/TypeOrPyobj.cpp @@ -24,11 +24,6 @@ TypeOrPyobj::TypeOrPyobj(Type* t) : if (!mType) { throw std::runtime_error("Can't construct a TypeOrPyobj with a null Type"); } - - if (mType->isForward() || mType->isForwardDefined()) { - asm("int3"); - throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); - } } TypeOrPyobj::TypeOrPyobj(PyObject* o) : @@ -41,11 +36,6 @@ TypeOrPyobj::TypeOrPyobj(PyObject* o) : mType = PyInstance::extractTypeFrom((PyTypeObject*)mPyObj, true); if (mType) { - if (mType->isForward() || mType->isForwardDefined()) { - asm("int3"); - throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); - } - mPyObj = nullptr; return; } @@ -69,11 +59,6 @@ TypeOrPyobj::TypeOrPyobj(PyTypeObject* o) : mType = PyInstance::extractTypeFrom(o, true); if (mType) { - if (mType->isForward() || mType->isForwardDefined()) { - asm("int3"); - throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); - } - mPyObj = nullptr; return; } @@ -96,11 +81,6 @@ TypeOrPyobj TypeOrPyobj::withoutIntern(PyObject* o) { if (PyType_Check(o)) { obj.mType = PyInstance::extractTypeFrom((PyTypeObject*)o, true); if (obj.mType) { - if (obj.mType->isForward() || obj.mType->isForwardDefined()) { - asm("int3"); - throw std::runtime_error("Can't construct a TOPO on a Forward-defined type"); - } - obj.mPyObj = nullptr; return obj; } diff --git a/typed_python/TypedCellType.hpp b/typed_python/TypedCellType.hpp index f39bc5180..e18b854e4 100644 --- a/typed_python/TypedCellType.hpp +++ b/typed_python/TypedCellType.hpp @@ -40,16 +40,17 @@ class TypedCellType : public Type { mHeldType(heldType) { m_is_forward_defined = true; - - m_size = sizeof(layout*); - - m_is_default_constructible = true; + recomputeName(); } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return "TypedCell(" + mHeldType->computeRecursiveName(typeStack) + ")"; } + const char* docConcrete() { + return "TypedCell: a cell held in a fully-typed function closure."; + } + void initializeDuringDeserialization(Type* heldType) { mHeldType = heldType; } @@ -62,7 +63,10 @@ class TypedCellType : public Type { updateTypeRefFromGroupMap(mHeldType, groupMap); } - void postInitializeConcrete() {} + void postInitializeConcrete() { + m_size = sizeof(layout*); + m_is_default_constructible = true; + } Type* cloneForForwardResolutionConcrete() { return new TypedCellType(); diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 31f99a60e..f3f15348a 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -41,7 +41,6 @@ from typed_python.type_function import TypeFunction from typed_python.hash import sha_hash from typed_python.SerializationContext import SerializationContext -from typed_python.type_filter import TypeFilter from typed_python.compiler.typeof import TypeOf from typed_python._types import ( Forward, TupleOf, ListOf, Tuple, NamedTuple, OneOf, ConstDict, SubclassOf, @@ -74,27 +73,27 @@ EmbeddedMessage = _types.EmbeddedMessage() PyCell = _types.PyCell() -from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa +# from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa -from typed_python.generator import Generator # noqa +# from typed_python.generator import Generator # noqa -# this has to come at the end to break import cyclic -from typed_python.lib.map import map # noqa -from typed_python.lib.pmap import pmap # noqa -from typed_python.lib.reduce import reduce # noqa +# # this has to come at the end to break import cyclic +# from typed_python.lib.map import map # noqa +# from typed_python.lib.pmap import pmap # noqa +# from typed_python.lib.reduce import reduce # noqa -_types.initializeGlobalStatics() +# _types.initializeGlobalStatics() -# start a background thread to release the GIL for us. Instead of immediately releasing the GIL, -# we prefer to release it a short time after our C code no longer needs it, in case it -# reacquires it immediately, which is very slow. -# this is needed primarily because we often can't tell whether we're about to enter code -# that acquires and releases the GIL very frequently (say, in a tight loop), which has -# a huge performance penalty (a few thousand context switches per second!) -# instead, we have a thread that checks in the background whether any thread wants us -# to release, and if so, swap it out. This can yield a 10-50x performance improvement -# when we're acquiring and releasing frequently. -gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) -gilReleaseThreadLoop.start() +# # start a background thread to release the GIL for us. Instead of immediately releasing the GIL, +# # we prefer to release it a short time after our C code no longer needs it, in case it +# # reacquires it immediately, which is very slow. +# # this is needed primarily because we often can't tell whether we're about to enter code +# # that acquires and releases the GIL very frequently (say, in a tight loop), which has +# # a huge performance penalty (a few thousand context switches per second!) +# # instead, we have a thread that checks in the background whether any thread wants us +# # to release, and if so, swap it out. This can yield a 10-50x performance improvement +# # when we're acquiring and releasing frequently. +# gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) +# gilReleaseThreadLoop.start() -_types.setGilReleaseThreadLoopSleepMicroseconds(500) +# _types.setGilReleaseThreadLoopSleepMicroseconds(500) diff --git a/typed_python/python_ast.py b/typed_python/python_ast.py index e5037152f..c2d6b62f3 100644 --- a/typed_python/python_ast.py +++ b/typed_python/python_ast.py @@ -1014,6 +1014,16 @@ def convertPyAstToAlgebraic(tree, fname, keepLineInformation=True): try: return converter(**args) except Exception: + from typed_python._types import identityHash + print("BOO!") + print(type(args['annotation'])) + print(converter.ElementType.ElementTypes[1]) + print(converter.ElementType.ElementTypes[1].Types) + print(identityHash(converter.ElementType.ElementTypes[1]).hex()) + print(converter.ElementType.ElementTypes[1].Types[0]) + print(OneOf(Expr, None)) + print(OneOf(Expr, None) is converter.ElementType.ElementTypes[1]) + raise UserWarning( "Failed to construct %s from %s with arguments\n%s\n\n%s" % ( converter, @@ -1405,7 +1415,6 @@ def evaluateFunctionDefWithLocalsInCells(pyAst, globals, locals, stripAnnotation TypeIgnore = resolveForwardDefinedType(TypeIgnore) - # map Python AST types to our syntax-tree types converters = { ast.Module: Module.Module, diff --git a/typed_python/test_serialize_recursive_types.py b/typed_python/test_serialize_recursive_types.py new file mode 100644 index 000000000..3db7766c2 --- /dev/null +++ b/typed_python/test_serialize_recursive_types.py @@ -0,0 +1,115 @@ +# Copyright 2017-2023 typed_python Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typed_python.test_util import callFunctionInFreshProcess +from typed_python import ( + Alternative, SerializationContext, Forward, resolveForwardDefinedType, Function +) + + +s = SerializationContext() + + +def test_can_serialize_function(): + @Function + def f(x: int): + return x + 1 + + assert type(f) is type(s.deserialize(s.serialize(f))) + + +def test_can_serialize_recursive_alternative(): + def makeA(): + A = Forward("A") + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: self + ) + ) + + A = A.resolve() + + return A + + assert makeA() is makeA() + assert makeA() is s.deserialize(s.serialize(makeA())) + + +def test_can_serialize_recursive_alternative_out_of_proc(): + def makeA(): + A = Forward("A") + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: self + ) + ) + + A = A.resolve() + + return A + + A_external = callFunctionInFreshProcess(makeA) + + assert makeA() is A_external + + +def test_can_serialize_recursive_alternative_out_of_proc_with_ref_to_self(): + def makeA(): + A = Forward("A") + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: A + ) + ) + + A = A.resolve() + + return A + + A_external = callFunctionInFreshProcess(makeA) + + assert makeA() is A_external + + +def test_can_serialize_recursive_alternative_out_of_proc_with_ref_to_own_forward(): + def makeA(): + A = Forward("A") + A_fwd = A + A.define( + Alternative( + "A", + x=dict(a=A), + y=dict(), + f=lambda self: A, + forward=lambda self: A_fwd + ) + ) + + A = A.resolve() + + return A + + A_external = callFunctionInFreshProcess(makeA) + + assert makeA() is A_external diff --git a/typed_python/test_util.py b/typed_python/test_util.py index 091b92dbd..08410d063 100644 --- a/typed_python/test_util.py +++ b/typed_python/test_util.py @@ -15,6 +15,7 @@ import psutil import time import threading +import textwrap import tempfile import pickle import subprocess @@ -22,7 +23,7 @@ import os from typed_python import sha_hash -from typed_python import Entrypoint, SerializationContext +# from typed_python import Entrypoint, SerializationContext def currentMemUsageMb(residentOnly=True): @@ -94,7 +95,7 @@ def instantiateFiles(filesToWrite, tf): ) -def callFunctionInFreshProcess(func, argTup, compilerCacheDir=None, showStdout=False, extraEnvs={}): +def callFunctionInFreshProcess(func, argTup=(), compilerCacheDir=None, showStdout=False, extraEnvs={}): """Return the value of a function evaluated on some arguments in a subprocess. We use this to test the semantics of anonymous functions and classes in a process @@ -244,6 +245,14 @@ def __init__(self): self.dir = tempfile.TemporaryDirectory() def evaluateInto(self, code, moduleDict): + # strip any blank lines at the end of 'code' and dedent it + codeLines = code.split("\n") + while codeLines and not codeLines[-1].strip(): + codeLines = codeLines[:-1] + code = "\n".join(codeLines) + + code = textwrap.dedent(code) + filename = os.path.abspath( os.path.join(self.dir.name, sha_hash(code).hexdigest) ) diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index a7a7120b6..a829d7408 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -6,6 +6,31 @@ Class, Member, ConstDict, TypedCell, Final ) +from typed_python.test_util import CodeEvaluator + + +def test_identity_hash_of_alternative_stable(): + e = CodeEvaluator() + m = {} + + e.evaluateInto(""" + from typed_python._types import identityHash, Alternative + + A = Alternative( + "A", + f=lambda x: (A, g) + ) + + h1 = identityHash(A) + + def g(): + pass + + h2 = identityHash(A) + """, m) + + assert m['h1'] == m['h2'] + def test_nonforward_definition(): assert not isForwardDefined(OneOf(int, float)) @@ -65,6 +90,11 @@ def test_forward_one_of(): assert bytecount(O_resolved) == 1 + bytecount(float) +def test_oneof(): + assert OneOf(int, float) is OneOf(int, float) + assert OneOf(int, float) is not OneOf(int, None) + + def test_recursive_definition_of_self(): F = Forward() @@ -398,6 +428,37 @@ class CChild(CBase): assert makeClass(str)().m == "" +def test_identity_of_leaked_forward(): + def makeClass(): + A = Forward("A") + A_fwd = A + + A.define( + Alternative( + "A", + X={}, + getA=lambda self: A, + getAForward=lambda self: A_fwd + ) + ) + A = resolveForwardDefinedType(A) + + return A, A_fwd + + A1, A1_fwd = makeClass() + A2, A2_fwd = makeClass() + + # the identity of 'A' should be stable + assert A1 is A2 + + # but the forwards we used to create them could be separate + assert A1_fwd is not A2_fwd + + # they have to point to the same concrete instance + assert A1_fwd.get() is A2_fwd.get() + assert A1_fwd.get() is A1 + + @pytest.mark.skip(reason='TODO: make this work') def test_create_class_with_fully_forward_base(): def makeClass(T): @@ -440,3 +501,4 @@ class CChild(C_base_fwd): # part of the graph being new, part being old # 8. clean-up in the whole Instance/PyInstance layer - kind of nasty # 9. we should memoize instances of statless Function objects and instances of regular TP lists/tuple of, etc. + diff --git a/typed_python/type_filter.py b/typed_python/type_filter.py deleted file mode 100644 index b697cf831..000000000 --- a/typed_python/type_filter.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2017-2019 typed_python Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class TypeFilterBase(type): - def __instancecheck__(self, other): - if not isinstance(other, self.base_type): - return False - - try: - if not self.filter_function(other): - return False - except Exception: - return False - - return True - - -def TypeFilter(base_type, filter_function): - """TypeFilter(base_type, filter_function) - - Produce a 'type object' that can be used in typed python to filter objects by - arbitrary criteria. - """ - class TypeFilter(metaclass=TypeFilterBase): - pass - - TypeFilter.base_type = base_type - TypeFilter.filter_function = filter_function - - return TypeFilter diff --git a/typed_python/type_identity_test.py b/typed_python/type_identity_test.py index fa61e0d34..d211e7535 100644 --- a/typed_python/type_identity_test.py +++ b/typed_python/type_identity_test.py @@ -639,13 +639,8 @@ def test_serialization_of_anonymous_functions_preserves_references(): def test_hash_stability(): idHash = evaluateExprInFreshProcess({ -<<<<<<< HEAD 'x.py': 'from typed_python.compiler.native_compiler.native_ast import NamedCallTarget\n' - }, 'identityHash(x.NamedCallTarget)') -======= - 'x.py': 'from typed_python.compiler.native_ast import NamedCallTarget\n' }, 'compilerHash(x.NamedCallTarget)') ->>>>>>> d4b04a6d... Add a 'VisibilityType' flag to MRTG. Refs #439. ser = returnSerializedValue({ 'x.py': 'from typed_python.compiler.native_compiler.native_ast import NamedCallTarget\n' }, 'x.NamedCallTarget', printComments=True) diff --git a/typed_python/types_test.py b/typed_python/types_test.py index 0e8142b8d..c3dcc3d56 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -31,7 +31,7 @@ Float32, SubclassOf, TupleOf, ListOf, OneOf, Tuple, NamedTuple, Dict, ConstDict, Alternative, serialize, deserialize, Class, - TypeFilter, Function, Forward, Set, PointerTo, + Function, Forward, Set, PointerTo, Entrypoint, Final, resolveForwardDefinedType @@ -535,24 +535,6 @@ def test_one_of_flattening(self): def test_one_of_order_matters(self): self.assertNotEqual(OneOf(1.0, 2.0), OneOf(2.0, 1.0)) - def test_type_filter(self): - EvenInt = TypeFilter(int, lambda i: i % 2 == 0) - - self.assertTrue(isinstance(2, EvenInt)) - self.assertFalse(isinstance(1, EvenInt)) - self.assertFalse(isinstance(2.0, EvenInt)) - - EvenIntegers = TupleOf(EvenInt) - - e = EvenIntegers(()) - e2 = e + (2, 4, 0) - - with self.assertRaises(TypeError): - EvenIntegers((1,)) - - with self.assertRaises(TypeError): - e2 + (1,) - def test_tuple_of_one_of_fixed_size(self): t = TupleOf(OneOf(0, 1, 2, 3, 4)) From 889579ec003f53ae4a1b13050448964012c40db8 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 13 Jun 2023 20:44:50 +0000 Subject: [PATCH 27/83] Start to work out a more refined notion of forwards --- typed_python/AlternativeType.cpp | 2 +- typed_python/FunctionType.hpp | 64 +++++++++++++++++++--- typed_python/HeldClassType.cpp | 8 +-- typed_python/Type.cpp | 75 ++++++++++++++++++++------ typed_python/Type.hpp | 13 +++-- typed_python/ValueType.cpp | 14 +++++ typed_python/__init__.py | 1 + typed_python/_types.cpp | 52 ++++++++++++++---- typed_python/type_construction_test.py | 71 +++++++++++++++++++++++- 9 files changed, 255 insertions(+), 45 deletions(-) diff --git a/typed_python/AlternativeType.cpp b/typed_python/AlternativeType.cpp index 35828ca8b..6599e1a19 100644 --- a/typed_python/AlternativeType.cpp +++ b/typed_python/AlternativeType.cpp @@ -178,7 +178,7 @@ Alternative* Alternative::Make( } for (auto nameAndMeth: methods) { - if (nameAndMeth.second->isForwardDefined()) { + if (nameAndMeth.second->isForwardDefined() || nameAndMeth.second->hasUnresolvedSymbols()) { anyForward = true; } } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index e9233b3d8..2f78c8a22 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -915,7 +915,6 @@ class Function : public Type { visitor.visitHash(ShaHash(1)); } - template static void visitCompilerVisibleGlobals( const visitor_type& visitor, @@ -1064,7 +1063,7 @@ class Function : public Type { return mFunctionGlobals; } - const std::map getFunctionGlobalsInCells() const { + const std::map& getFunctionGlobalsInCells() const { return mFunctionGlobalsInCells; } @@ -1201,6 +1200,43 @@ class Function : public Type { mFunctionGlobals = incref(globals); } + bool symbolIsUnresolved(std::string name) { + if (mClosureBindings.find(name) != mClosureBindings.end()) { + return false; + } + + auto it = mFunctionGlobalsInCells.find(name); + if (it != mFunctionGlobalsInCells.end()) { + if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + return false; + } + } + + if (mFunctionGlobals && PyDict_Check(mFunctionGlobals) + && PyDict_GetItemString(mFunctionGlobals, name.c_str())) { + return false; + } + + return true; + } + + bool hasUnresolvedSymbols() { + std::set allNames; + extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); + + for (auto name: allNames) { + if (PyUnicode_Check(name)) { + std::string nameStr = PyUnicode_AsUTF8(name); + + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr)) { + return true; + } + } + } + + return false; + } + PyObject* getUsedGlobals() const { // restrict the globals to contain only the values it references. PyObject* result = PyDict_New(); @@ -1579,6 +1615,16 @@ class Function : public Type { return mModulename + "." + mRootName; } + bool hasUnresolvedSymbols() { + for (auto& o: mOverloads) { + if (o.hasUnresolvedSymbols()) { + return true; + } + } + + return false; + } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return mRootName; } @@ -1653,6 +1699,10 @@ class Function : public Type { anyFwd = true; } }); + + if (o.hasUnresolvedSymbols()) { + anyFwd = true; + } } if (anyFwd) { @@ -1937,11 +1987,11 @@ class Function : public Type { return true; } - //Function types can be instantiated even if forwards are not resolved - //in their annotations. - void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { - mClosureType->assertForwardsResolvedSufficientlyToInstantiate(); - } + // //Function types can be instantiated even if forwards are not resolved + // //in their annotations. + // void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { + // mClosureType->assertForwardsResolvedSufficientlyToInstantiate(); + // } Function* replaceClosure(Type* closureType) const { return Function::Make(mRootName, mQualname, mModulename, mOverloads, closureType, mIsEntrypoint, mIsNocompile); diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index 8e4ddcd9f..a39334055 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -491,17 +491,17 @@ HeldClass* HeldClass::Make( } for (auto& nameAndT: memberFunctions) { - if (nameAndT.second->isForwardDefined()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { anyForward = true; } } for (auto& nameAndT: staticFunctions) { - if (nameAndT.second->isForwardDefined()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { anyForward = true; } } for (auto& nameAndT: propertyFunctions) { - if (nameAndT.second->isForwardDefined()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { anyForward = true; } } @@ -514,7 +514,7 @@ HeldClass* HeldClass::Make( // } for (auto& nameAndT: classMethods) { - if (nameAndT.second->isForwardDefined()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { anyForward = true; } } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 8bf82b21a..ed5687e37 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -254,38 +254,79 @@ bool Type::canConvertToTrivially(Type* otherType) { return false; } -// try to resolve this forward type. If we can't, we'll throw an exception. On exit, -// we will have thrown, or m_forward_resolves_to will be populated. -void Type::attemptToResolve() { - if (m_forward_resolves_to) { - return; - } - - // do the entire type resolution process while holding the GIL - PyEnsureGilAcquired getTheGil; - - std::set existingReferencedTypes; - std::set typesNeedingResolution; +void reachableUnresolvedTypes(Type* root, std::set& outTypes) { std::set toVisit; - toVisit.insert(this); + toVisit.insert(root); // walk the graph and determine all forwards that are not resolved while (toVisit.size()) { Type* toCheck = *toVisit.begin(); toVisit.erase(toCheck); - if (toCheck->isForwardDefined() && !toCheck->m_forward_resolves_to) { - if (typesNeedingResolution.find(toCheck) == typesNeedingResolution.end()) { - typesNeedingResolution.insert(toCheck); + if (toCheck->isForwardDefined() && !toCheck->isResolved()) { + if (outTypes.find(toCheck) == outTypes.end()) { + outTypes.insert(toCheck); toCheck->visitReferencedTypes([&](Type* subtype) { - if (subtype->isForwardDefined() && !subtype->m_forward_resolves_to) { + if (subtype->isForwardDefined() && !subtype->isResolved()) { toVisit.insert(subtype); } }); } } } +} + +bool Type::looksResolvable(bool unambiguously) { + if (!m_is_forward_defined) { + return true; + } + + if (m_forward_resolves_to) { + return true; + } + + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget()) { + return false; + } + } + + if (unambiguously) { + for (auto t: typesNeedingResolution) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols()) { + return false; + } + } + } + + return true; +} + +// try to resolve this forward type. If we can't, we'll throw an exception. On exit, +// we will have thrown, or m_forward_resolves_to will be populated. +void Type::attemptToResolve() { + if (m_forward_resolves_to) { + return; + } + + if (!m_is_forward_defined) { + return; + } + + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + std::set existingReferencedTypes; for (auto t: typesNeedingResolution) { if (t->isForward() && !((Forward*)t)->getTarget()) { diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 283caef4f..59d54d258 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -886,6 +886,16 @@ class Type { return m_is_forward_defined; } + bool isResolved() const { + if (!m_is_forward_defined) { + return true; + } + + return m_forward_resolves_to != nullptr; + } + + bool looksResolvable(bool unambiguously); + Type* forwardResolvesTo() { if (!m_is_forward_defined) { return this; @@ -913,9 +923,6 @@ class Type { m_is_redundant(false) {} - //todo: remove this - bool m_is_recursive_forward; - TypeCategory m_typeCategory; size_t m_size; diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 1161e16a8..34b115ffd 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -34,6 +34,12 @@ Value::Value(const Instance& instance) : m_is_default_constructible = true; mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } + m_is_forward_defined = true; } @@ -125,10 +131,18 @@ void Value::updateInternalTypePointersConcrete( (PyObject*)PyInstance::typeObj(it->second) ); mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } } } void Value::initializeFromConcrete(Type* forwardDefinitionOfSelf) { mInstance = ((Value*)forwardDefinitionOfSelf)->mInstance.clone(); mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } } diff --git a/typed_python/__init__.py b/typed_python/__init__.py index f3f15348a..4b2ec7c64 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -54,6 +54,7 @@ ModuleRepresentation, setGilReleaseThreadLoopSleepMicroseconds, isForwardDefined, + typeLooksResolvable, resolveForwardDefinedType, identityHash ) diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 4efea9bc4..dd44586f3 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2505,10 +2505,10 @@ PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { PyObjectHolder a1(PyTuple_GetItem(args, 0)); return translateExceptionToPyObject([&]() { - Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); - if (typeOfArg && typeOfArg->isFunction() && typeOfArg->bytecount() == 0) { - return incref(typeOfArg->isForwardDefined() ? Py_True : Py_False); - } + // Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->isForwardDefined() ? Py_True : Py_False); + // } Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); @@ -2520,6 +2520,33 @@ PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { }); } +PyObject *typeLooksResolvable(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* type; + int unambiguously = 0; + + static const char *kwlist[] = {"type", "unambiguously", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|p", (char**)kwlist, &type, &unambiguously)) { + throw PythonExceptionSet(); + } + + // see first if its an unclosured function object + // Type* typeOfArg = PyInstance::extractTypeFrom(type->ob_type, false); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->looksResolvable(unambiguously) ? Py_True : Py_False); + // } + + Type* t = PyInstance::unwrapTypeArgToTypePtr(type); + + if (!t) { + throw std::runtime_error("type must be a TP type instance"); + } + + return incref(t->looksResolvable(unambiguously) ? Py_True : Py_False); + }); +} + PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "resolveForwardDefinedType takes 1 positional argument"); @@ -2527,15 +2554,17 @@ PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { } PyObjectHolder a1(PyTuple_GetItem(args, 0)); - Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + // Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type, false); - if (typeOfArg && typeOfArg->isFunction() && typeOfArg->bytecount() == 0) { - return ::translateExceptionToPyObject([&]() { - Type* newType = typeOfArg->forwardResolvesTo(); + // if (typeOfArg && typeOfArg->isFunction()) { + // return ::translateExceptionToPyObject([&]() { + // Type* newType = typeOfArg->forwardResolvesTo(); - return PyInstance::initialize(newType, [&](instance_ptr data) {}); - }); - } + // return PyInstance::initialize(newType, [&](instance_ptr data) { + // ((Function*)newType)->getClosureType()->copy_constructor(data, ((PyInstance*)(PyObject*)a1)->dataPtr()); + // }); + // }); + // } Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); @@ -3473,6 +3502,7 @@ static PyMethodDef module_methods[] = { {"isPOD", (PyCFunction)isPOD, METH_VARARGS, NULL}, {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, + {"typeLooksResolvable", (PyCFunction)typeLooksResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index a829d7408..eec76edfe 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, Final + Class, Member, ConstDict, TypedCell, Final, typeLooksResolvable ) from typed_python.test_util import CodeEvaluator @@ -32,6 +32,74 @@ def g(): assert m['h1'] == m['h2'] +def test_type_looks_resolvable_alternative(): + # note that these types are not autoresolvable because of their names + A = Alternative("A_", X={}, f=lambda self: B) + + assert typeLooksResolvable(A, unambiguously=False) + assert not typeLooksResolvable(A, unambiguously=True) + + B = Alternative("B_", X={}, g=lambda self: A) + + assert typeLooksResolvable(A, unambiguously=False) + assert typeLooksResolvable(A, unambiguously=True) + assert typeLooksResolvable(B, unambiguously=False) + assert typeLooksResolvable(B, unambiguously=True) + + A = resolveForwardDefinedType(A) + B = resolveForwardDefinedType(B) + + assert A().f() is B + assert B().g() is A + + +def test_untyped_functions_are_not_forward(): + # this type is not forward defined because it should be holding 'g' in the closure of + # 'f' itself + f = Function(lambda: g()) + assert not isForwardDefined(type(f)) + + g = Function(lambda: f()) + assert not isForwardDefined(type(g)) + + +def test_recursive_types(): + # define a forward + O = Forward("O") + + # give the forward a definition. the thing that comes out is the resolved type + O = O.define(ListOf(OneOf(int, O))) + + +def test_dual_recursive_types_direct_instantiation(): + O1 = Forward("O1") + O2 = Forward("O2") + + O1.define(ListOf(OneOf(int, O2))) + O2.define(ListOf(OneOf(int, O1))) + + O1 = resolveForwardDefinedType(O1) + O2 = resolveForwardDefinedType(O2) + + +def test_dual_recursive_types_direct_instantiation_with_alternatives(): + O1 = Forward("O1") + O2 = Forward("O2") + + O1.define(Alternative("O1", X={}, Y={'a': O2}, f=lambda self: O2)) + O2.define(Alternative("O2", X={}, Y={'a': O1}, f=lambda self: O1)) + + O1 = resolveForwardDefinedType(O1) + O2 = resolveForwardDefinedType(O2) + + +def test_dual_recursive_types_implicit_instantiation_with_alternatives(): + O1 = Alternative("O1", X={}, Y={'a': lambda: O2}, f=lambda self: O2) + + # this assignment triggers a resolution + O2 = Alternative("O2", X={}, Y={'a': lambda: O1}, f=lambda self: O1) + + def test_nonforward_definition(): assert not isForwardDefined(OneOf(int, float)) assert not isForwardDefined(ListOf(int)) @@ -501,4 +569,3 @@ class CChild(C_base_fwd): # part of the graph being new, part being old # 8. clean-up in the whole Instance/PyInstance layer - kind of nasty # 9. we should memoize instances of statless Function objects and instances of regular TP lists/tuple of, etc. - From 7a099685fc1fadf2db682306c64e85d9dcb51109 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 13 Jun 2023 22:08:42 +0000 Subject: [PATCH 28/83] Forwards can accept simple lambdas. --- typed_python/ForwardType.cpp | 41 +++++++++++ typed_python/ForwardType.hpp | 96 ++++++++++++++++++++++++-- typed_python/Type.cpp | 12 +++- typed_python/_types.cpp | 50 +++++++------- typed_python/all.cpp | 1 + typed_python/type_construction_test.py | 28 ++++++-- 6 files changed, 191 insertions(+), 37 deletions(-) create mode 100644 typed_python/ForwardType.cpp diff --git a/typed_python/ForwardType.cpp b/typed_python/ForwardType.cpp new file mode 100644 index 000000000..d498ea1e2 --- /dev/null +++ b/typed_python/ForwardType.cpp @@ -0,0 +1,41 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyInstance.hpp" +#include "ForwardType.hpp" + + +Type* Forward::lambdaDefinition() { + if (!mCellOrDict) { + return nullptr; + } + + if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { + return PyInstance::unwrapTypeArgToTypePtr(PyCell_Get(mCellOrDict)); + } + + if (PyDict_Check(mCellOrDict)) { + PyObject* item = PyDict_GetItemString(mCellOrDict, m_name.c_str()); + + if (!item) { + return nullptr; + } + + return PyInstance::unwrapTypeArgToTypePtr(item); + } + + return nullptr; +} diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 07d6187a5..3f02f863d 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2019 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,9 +29,10 @@ PyDoc_STRVAR(Forward_doc, // any types that contain them can be used. class Forward : public Type { public: - Forward(std::string name) : + Forward(std::string name, PyObject* cellOrDict=nullptr) : Type(TypeCategory::catForward), - mTarget(nullptr) + mTarget(nullptr), + mCellOrDict(incref(cellOrDict)) { m_name = name; m_is_forward_defined = true; @@ -65,6 +66,65 @@ class Forward : public Type { return new Forward(name); } + static Forward* MakeFromFunction(PyObject* funcObj) { + if (!PyFunction_Check(funcObj)) { + throw std::runtime_error( + "Forwards can only be made from functions which return a single variable lookup." + ); + } + + PyCodeObject* code = (PyCodeObject*)PyFunction_GetCode(funcObj); + PyObject* closure = PyFunction_GetClosure(funcObj); + + if (!PyTuple_Check(code->co_names)) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names is not a tuple"); + } + + if (closure) { + PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); + + if (PyTuple_Size(code->co_names)) { + throw std::runtime_error("Corrupt Forward lookup function: can't have more than one name"); + } + + if (!PyTuple_Check(coFreevars)) { + throw std::runtime_error("Corrupt Forward lookup function: co_freevars was not a tuple"); + } + + if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { + throw std::runtime_error("Corrupt Forward lookup function: co_freevars not the same size as closure"); + } + + if (PyTuple_Size(coFreevars) != 1) { + throw std::runtime_error("Corrupt Forward lookup function: can't have more than 1 free variable"); + } + + PyObject* varname = PyTuple_GetItem(coFreevars, 0); + if (!PyUnicode_Check(varname)) { + throw std::runtime_error("Corrupt Forward lookup function: closure varnames need to be strings"); + } + + std::string name = PyUnicode_AsUTF8(varname); + + if (!PyCell_Check(PyTuple_GetItem(closure, 0))) { + throw std::runtime_error("Corrupt Forward lookup function: closure was not a cell"); + } + + return new Forward(name, PyTuple_GetItem(closure, 0)); + } + + if (PyTuple_Size(code->co_names) != 1) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); + } + + if (!PyUnicode_Check(PyTuple_GetItem(code->co_names, 0))) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); + } + + std::string name = PyUnicode_AsUTF8(PyTuple_GetItem(code->co_names, 0)); + return new Forward(name, PyFunction_GetGlobals(funcObj)); + } + Type* getTargetTransitive() { if (!mTarget) { return nullptr; @@ -126,10 +186,33 @@ class Forward : public Type { template void _visitReferencedTypes(const visitor_type& v) { - if (mTarget) + if (mTarget) { v(mTarget); + } + } + + bool hasLambdaDefinition() { + return mCellOrDict != nullptr; } + bool lambdaDefinitionPopulated() { + if (!mCellOrDict) { + return false; + } + + if (PyCell_Check(mCellOrDict)) { + return PyCell_Get(mCellOrDict) != nullptr; + } + + if (PyDict_Check(mCellOrDict)) { + return PyDict_GetItemString(mCellOrDict, m_name.c_str()); + } + + return false; + } + + Type* lambdaDefinition(); + void constructor(instance_ptr self) { throw std::runtime_error("Forward types should never be explicity instantiated."); } @@ -158,4 +241,9 @@ class Forward : public Type { private: Type* mTarget; + + // if we are a recursive forward defined by a simple lambda function encoded by name, + // then this will contain either a PyCell from the closure, or a PyDict in which + // we are supposed to look to find a 'name'. + PyObject* mCellOrDict; }; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index ed5687e37..37cf7d92f 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -293,7 +293,7 @@ bool Type::looksResolvable(bool unambiguously) { reachableUnresolvedTypes(this, typesNeedingResolution); for (auto t: typesNeedingResolution) { - if (t->isForward() && !((Forward*)t)->getTarget()) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { return false; } } @@ -329,13 +329,21 @@ void Type::attemptToResolve() { std::set existingReferencedTypes; for (auto t: typesNeedingResolution) { - if (t->isForward() && !((Forward*)t)->getTarget()) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { throw std::runtime_error( "Forward defined as " + t->nameWithModule() + " has not been defined." ); } } + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget()) { + ((Forward*)t)->define( + ((Forward*)t)->lambdaDefinition() + ); + } + } + // for each type that we have defined in our graph, what type are we mapping it to? // for Forwards of primitive types like ListOf or TupleOf that can see themselves, this // this will be a 'named copy' of the type depending on the Forward name. diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index dd44586f3..f49e4a907 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -3134,39 +3134,37 @@ PyObject *referencedTypes(PyObject* nullValue, PyObject* args) { } PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs) { - int num_args = PyTuple_Size(args); - - PyThreadState * ts = PyThreadState_Get(); - std::string moduleName; - PyObject* pyModuleName; + return translateExceptionToPyObject([&]() { + int num_args = PyTuple_Size(args); - if (ts->frame && ts->frame->f_globals && - (pyModuleName = PyDict_GetItemString(ts->frame->f_globals, "__name__"))) { - if (PyUnicode_Check(pyModuleName)) { - moduleName = PyUnicode_AsUTF8(pyModuleName); + if (num_args == 1 && PyUnicode_Check(PyTuple_GetItem(args, 0))) { + return incref((PyObject*)PyInstance::typeObj( + Forward::Make( + PyUnicode_AsUTF8(PyTuple_GetItem(args,0)) + ) + )); } - } - if (num_args > 1 || (num_args == 1 && !PyUnicode_Check(PyTuple_GetItem(args,0)))) { - PyErr_SetString(PyExc_TypeError, "Forward takes a zero or one string positional arguments."); - return NULL; - } + if (num_args == 1 && PyFunction_Check(PyTuple_GetItem(args, 0))) { + return incref((PyObject*)PyInstance::typeObj( + Forward::MakeFromFunction(PyTuple_GetItem(args, 0)) + )); + } - if (num_args == 1) { - std::string name = PyUnicode_AsUTF8(PyTuple_GetItem(args,0)); + if (num_args == 0) { + return incref((PyObject*)PyInstance::typeObj( + Forward::Make() + )); + } - if (moduleName.size()) { - name = moduleName + "." + name; + if (num_args > 1) { + throw std::runtime_error("Can't make a Forward with more than 1 argument."); } - return incref((PyObject*)PyInstance::typeObj( - ::Forward::Make(name) - )); - } else { - return incref((PyObject*)PyInstance::typeObj( - ::Forward::Make() - )); - } + throw std::runtime_error( + "Forward requires either a string or a function with a simple name lookup." + ); + }); } /********* diff --git a/typed_python/all.cpp b/typed_python/all.cpp index 6f0f64b3b..76acff4ee 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -61,6 +61,7 @@ compile the entire group all at once. #include "TupleOrListOfType.cpp" #include "Type.cpp" #include "FunctionType.cpp" +#include "ForwardType.cpp" #include "ValueType.cpp" #include "SerializationBuffer.cpp" diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index eec76edfe..8276350bf 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -63,12 +63,30 @@ def test_untyped_functions_are_not_forward(): assert not isForwardDefined(type(g)) -def test_recursive_types(): - # define a forward - O = Forward("O") +def test_invalid_forwards(): + with pytest.raises(TypeError): + Forward(lambda: 1) + + with pytest.raises(TypeError): + Forward(lambda: A + B) # noqa + + X = 10 + with pytest.raises(TypeError): + Forward(lambda: X + B) # noqa + + +def test_implicitly_recursive_types(): + O_fwd = ListOf(OneOf(int, Forward(lambda: O))) + + assert isForwardDefined(O_fwd) + assert not typeLooksResolvable(O_fwd) + + O = O_fwd + + assert typeLooksResolvable(O_fwd) - # give the forward a definition. the thing that comes out is the resolved type - O = O.define(ListOf(OneOf(int, O))) + O = resolveForwardDefinedType(O) + O([O()]) def test_dual_recursive_types_direct_instantiation(): From f2de8b674e315f27216765e389ef0f2fffbceb8a Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 14 Jun 2023 05:13:11 +0000 Subject: [PATCH 29/83] Sort-of working autoresolution. --- typed_python/AlternativeType.cpp | 2 +- typed_python/ForwardType.cpp | 58 +++++++- typed_python/ForwardType.hpp | 6 + typed_python/FunctionType.hpp | 108 ++++++++++++++- typed_python/HeldClassType.cpp | 8 +- typed_python/OneOfType.cpp | 2 +- typed_python/PyInstance.cpp | 9 ++ typed_python/PyInstance.hpp | 8 ++ typed_python/PyTemporaryReferenceTracer.cpp | 55 +++++++- typed_python/PyTemporaryReferenceTracer.hpp | 19 ++- ...onSerializationContext_deserialization.cpp | 2 +- typed_python/Type.cpp | 47 ++++++- typed_python/Type.hpp | 12 +- typed_python/TypeOrPyobj.cpp | 3 +- typed_python/_types.cpp | 128 ++++++++++++++---- typed_python/python_ast.py | 10 -- typed_python/type_construction_test.py | 55 ++++++++ 17 files changed, 468 insertions(+), 64 deletions(-) diff --git a/typed_python/AlternativeType.cpp b/typed_python/AlternativeType.cpp index 6599e1a19..6e270143a 100644 --- a/typed_python/AlternativeType.cpp +++ b/typed_python/AlternativeType.cpp @@ -178,7 +178,7 @@ Alternative* Alternative::Make( } for (auto nameAndMeth: methods) { - if (nameAndMeth.second->isForwardDefined() || nameAndMeth.second->hasUnresolvedSymbols()) { + if (nameAndMeth.second->isForwardDefined() || nameAndMeth.second->hasUnresolvedSymbols(true)) { anyForward = true; } } diff --git a/typed_python/ForwardType.cpp b/typed_python/ForwardType.cpp index d498ea1e2..22713728a 100644 --- a/typed_python/ForwardType.cpp +++ b/typed_python/ForwardType.cpp @@ -23,7 +23,7 @@ Type* Forward::lambdaDefinition() { return nullptr; } - if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { + if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { return PyInstance::unwrapTypeArgToTypePtr(PyCell_Get(mCellOrDict)); } @@ -39,3 +39,59 @@ Type* Forward::lambdaDefinition() { return nullptr; } + +void Forward::installDefinitionIfLambda() { + if (!mCellOrDict) { + return; + } + + if (!m_forward_resolves_to) { + return; + } + + if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { + if (PyType_Check(PyCell_Get(mCellOrDict))) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)PyCell_Get(mCellOrDict) + ); + + if (cellContents) { + // see if cellContents resolves + if (cellContents->isResolved() + && cellContents->forwardResolvesTo() + == m_forward_resolves_to) { + PyCell_Set( + mCellOrDict, + (PyObject*)PyInstance::typeObj( + m_forward_resolves_to + ) + ); + } + } + } + + return; + } + + if (PyDict_Check(mCellOrDict)) { + PyObject* curContents = PyDict_GetItemString(mCellOrDict, m_name.c_str()); + + if (curContents && PyType_Check(curContents)) { + Type* curContentsType = PyInstance::extractTypeFrom( + (PyTypeObject*)curContents + ); + + if (curContentsType && curContentsType->isResolved() && + curContentsType->forwardResolvesTo() + == m_forward_resolves_to) { + PyDict_SetItemString( + mCellOrDict, + m_name.c_str(), + (PyObject*)PyInstance::typeObj( + m_forward_resolves_to + ) + ); + } + } + } +} diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 3f02f863d..a296b160e 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -189,6 +189,10 @@ class Forward : public Type { if (mTarget) { v(mTarget); } + + if (lambdaDefinitionPopulated()) { + v(lambdaDefinition()); + } } bool hasLambdaDefinition() { @@ -213,6 +217,8 @@ class Forward : public Type { Type* lambdaDefinition(); + void installDefinitionIfLambda(); + void constructor(instance_ptr self) { throw std::runtime_error("Forward types should never be explicity instantiated."); } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 2f78c8a22..976caa614 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1200,7 +1200,7 @@ class Function : public Type { mFunctionGlobals = incref(globals); } - bool symbolIsUnresolved(std::string name) { + bool symbolIsUnresolved(std::string name, bool insistResolved) { if (mClosureBindings.find(name) != mClosureBindings.end()) { return false; } @@ -1208,19 +1208,105 @@ class Function : public Type { auto it = mFunctionGlobalsInCells.find(name); if (it != mFunctionGlobalsInCells.end()) { if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + if (insistResolved) { + PyObject* o = PyCell_Get(it->second); + Type* t = PyInstance::extractTypeFrom(o); + if (t && t->isForwardDefined()) { + return true; + } + } + return false; } + + return true; } if (mFunctionGlobals && PyDict_Check(mFunctionGlobals) && PyDict_GetItemString(mFunctionGlobals, name.c_str())) { + if (insistResolved) { + PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + Type* t = PyInstance::extractTypeFrom(o); + if (t && t->isForwardDefined()) { + return true; + } + } + return false; } return true; } - bool hasUnresolvedSymbols() { + void autoresolveGlobal( + std::string name, + const std::set& resolvedForwards + ) { + std::cout << "want to autoresolve " << name << "\n"; + + auto it = mFunctionGlobalsInCells.find(name); + if (it != mFunctionGlobalsInCells.end()) { + std::cout << "considering " << name << "\n"; + if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + if (PyType_Check(PyCell_GET(it->second))) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)PyCell_Get(it->second) + ); + + std::cout << "look at " << TypeOrPyobj(cellContents).name() << "\n"; + + if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + std::cout << "autoresvole " << name << " as cell\n"; + + PyCell_Set( + it->second, + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + } + } + + return; + } + + PyObject* dictVal = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + if (dictVal && PyType_Check(dictVal)) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)dictVal + ); + + if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + std::cout << "autoresvole " << name << "\n"; + + PyDict_SetItemString( + mFunctionGlobals, + name.c_str(), + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + } + } + + void autoresolveGlobals(const std::set& resolvedForwards) { + std::set allNames; + extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); + + for (auto name: allNames) { + if (PyUnicode_Check(name)) { + std::string nameStr = PyUnicode_AsUTF8(name); + + if (nameStr != "__module_hash__") { + autoresolveGlobal(nameStr, resolvedForwards); + } + } + } + } + + bool hasUnresolvedSymbols(bool insistResolved) { std::set allNames; extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); @@ -1228,7 +1314,7 @@ class Function : public Type { if (PyUnicode_Check(name)) { std::string nameStr = PyUnicode_AsUTF8(name); - if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr)) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { return true; } } @@ -1615,9 +1701,13 @@ class Function : public Type { return mModulename + "." + mRootName; } - bool hasUnresolvedSymbols() { + // does this function have any of its globals that are not + // resolved to actual values. if 'insistResolved' then we + // also return 'true' if the symbols resolve to a forward defined + // type + bool hasUnresolvedSymbols(bool insistResolved) { for (auto& o: mOverloads) { - if (o.hasUnresolvedSymbols()) { + if (o.hasUnresolvedSymbols(insistResolved)) { return true; } } @@ -1625,6 +1715,12 @@ class Function : public Type { return false; } + void autoresolveGlobals(const std::set& resolvedForwards) { + for (auto& o: mOverloads) { + o.autoresolveGlobals(resolvedForwards); + } + } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return mRootName; } @@ -1700,7 +1796,7 @@ class Function : public Type { } }); - if (o.hasUnresolvedSymbols()) { + if (o.hasUnresolvedSymbols(true)) { anyFwd = true; } } diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index a39334055..7fae7a77f 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -491,17 +491,17 @@ HeldClass* HeldClass::Make( } for (auto& nameAndT: memberFunctions) { - if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { anyForward = true; } } for (auto& nameAndT: staticFunctions) { - if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { anyForward = true; } } for (auto& nameAndT: propertyFunctions) { - if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { anyForward = true; } } @@ -514,7 +514,7 @@ HeldClass* HeldClass::Make( // } for (auto& nameAndT: classMethods) { - if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols()) { + if (nameAndT.second->isForwardDefined() || nameAndT.second->hasUnresolvedSymbols(true)) { anyForward = true; } } diff --git a/typed_python/OneOfType.cpp b/typed_python/OneOfType.cpp index 650c8f855..2c83a7d58 100644 --- a/typed_python/OneOfType.cpp +++ b/typed_python/OneOfType.cpp @@ -145,7 +145,7 @@ void OneOfType::updateInternalTypePointersConcrete( } } else { auto it = groupMap.find(t); - if (it != groupMap.end()) { + if (it != groupMap.end() && it->second != t) { visit(it->second); } else { if (seenTypes.find(t) == seenTypes.end()) { diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 3ea72f3c8..227162dc0 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1784,6 +1784,14 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { ); } + if (PyFunction_Check(typearg)) { + try { + return Forward::MakeFromFunction(typearg); + } catch(std::exception& e) { + // do nothing + } + } + // else: typearg is not a type -> it is a value Type* valueType = PyInstance::tryUnwrapPyInstanceToValueType(typearg, false); @@ -1791,6 +1799,7 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { return valueType; } + PyErr_Format(PyExc_TypeError, "Cannot convert %R to a typed_python Type", typearg); return NULL; } diff --git a/typed_python/PyInstance.hpp b/typed_python/PyInstance.hpp index 9eaceb052..dc7403d0d 100644 --- a/typed_python/PyInstance.hpp +++ b/typed_python/PyInstance.hpp @@ -572,6 +572,14 @@ class PyInstance { // float and int, which happen to have both PyTypeObject* and Type* representations static Type* extractTypeFrom(PyTypeObject* typeObj, bool includePrimitives=false); + static Type* extractTypeFrom(PyObject* typeObj, bool includePrimitives=false) { + if (!PyType_Check(typeObj)) { + return nullptr; + } + + return extractTypeFrom((PyTypeObject*)typeObj, includePrimitives); + } + // if 'obj' is a PyInstance, return its type and data ptrs, after unwrapping // any 'RefTo' classes, which is the standard behavior for most classes, // which don't specifically know anything about 'RefTo' diff --git a/typed_python/PyTemporaryReferenceTracer.cpp b/typed_python/PyTemporaryReferenceTracer.cpp index 046436a71..0e93cd2f5 100644 --- a/typed_python/PyTemporaryReferenceTracer.cpp +++ b/typed_python/PyTemporaryReferenceTracer.cpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2021 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -80,9 +80,26 @@ int PyTemporaryReferenceTracer::globalTraceFun(PyObject* dummyObj, PyFrameObject if (frameAction.lineNumber != PyFrame_GetLineNumber(frame) || forceProcess) { if (frameAction.action == TraceAction::ConvertTemporaryReference) { ((PyInstance*)frameAction.obj)->resolveTemporaryReference(); + decref(frameAction.obj); + } + else if (frameAction.action == TraceAction::Decref) { + decref(frameAction.obj); + } + else if (frameAction.action == TraceAction::Autoresolve) { + PyErrorStasher stashCurrentException; + + // attempt to autoresolve the type. Note that if we + // fail to do so, we just swallow the exception, which + // is not great, but we can't be throwing exceptions + // in here... + try { + frameAction.typ->tryToAutoresolve(); + } catch(std::exception& e) { + // just swallow it + } catch(PythonExceptionSet& e) { + PyErr_Clear(); + } } - - decref(frameAction.obj); } else { persistingActions.push_back(frameAction); } @@ -163,6 +180,23 @@ void PyTemporaryReferenceTracer::keepaliveForCurrentInstruction(PyObject* o, PyF installGlobalTraceHandlerIfNecessary(); } +void PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* t, PyFrameObject* f) { + // mark that we're going to trace + globalTracer.frameToActions[f].push_back( + FrameAction( + t, + TraceAction::Autoresolve, + PyFrame_GetLineNumber(f) + ) + ); + + if (globalTracer.mostRecentEmptyFrame == f) { + globalTracer.mostRecentEmptyFrame = nullptr; + } + + installGlobalTraceHandlerIfNecessary(); +} + void PyTemporaryReferenceTracer::traceObject(PyObject* o) { PyThreadState *tstate = PyThreadState_GET(); PyFrameObject *f = tstate->frame; @@ -172,6 +206,21 @@ void PyTemporaryReferenceTracer::traceObject(PyObject* o) { } } +Type* PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* o) { + if (!o->isForwardDefined() || o->isResolved()) { + return o; + } + + PyThreadState *tstate = PyThreadState_GET(); + PyFrameObject *f = tstate->frame; + + if (f) { + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(o, f); + } + + return o; +} + void PyTemporaryReferenceTracer::keepaliveForCurrentInstruction(PyObject* o) { PyThreadState *tstate = PyThreadState_GET(); PyFrameObject *f = tstate->frame; diff --git a/typed_python/PyTemporaryReferenceTracer.hpp b/typed_python/PyTemporaryReferenceTracer.hpp index 75d6a278b..589b8bdeb 100644 --- a/typed_python/PyTemporaryReferenceTracer.hpp +++ b/typed_python/PyTemporaryReferenceTracer.hpp @@ -21,7 +21,8 @@ enum class TraceAction { ConvertTemporaryReference, - Decref + Decref, + Autoresolve }; class PyTemporaryReferenceTracer { @@ -39,12 +40,22 @@ class PyTemporaryReferenceTracer { public: FrameAction(PyObject* inO, TraceAction inA, int inLine) : obj(inO), + typ(nullptr), + action(inA), + lineNumber(inLine) + { + } + + FrameAction(Type* inT, TraceAction inA, int inLine) : + obj(nullptr), + typ(inT), action(inA), lineNumber(inLine) { } PyObject* obj; + Type* typ; TraceAction action; int lineNumber; }; @@ -72,6 +83,12 @@ class PyTemporaryReferenceTracer { static void traceObject(PyObject* o); + // on the next instruction of 'frame', attempt to autoresolve 't' + static void autoresolveOnNextInstruction(Type* t, PyFrameObject* frame); + + // on the next instruction of 'frame', attempt to autoresolve 't' + static Type* autoresolveOnNextInstruction(Type* t); + static void installGlobalTraceHandlerIfNecessary(); static void keepaliveForCurrentInstruction(PyObject* o); diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 8d095d64f..fc771bae6 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1405,7 +1405,7 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b ); } - Type* nativeType = PyInstance::extractTypeFrom(typeFromRep); + Type* nativeType = PyInstance::extractTypeFrom((PyTypeObject*)typeFromRep); if (!nativeType) { // the type representation could be a regular python class diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 37cf7d92f..b56fe88db 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -300,7 +300,7 @@ bool Type::looksResolvable(bool unambiguously) { if (unambiguously) { for (auto t: typesNeedingResolution) { - if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols()) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols(false)) { return false; } } @@ -500,6 +500,51 @@ void Type::internalize() { mInternalizedIdentityHashToType[mIdentityHash] = this; } +void Type::tryToAutoresolve() { + // some types don't need resolution + if (!m_is_forward_defined) { + return; + } + + // some types are already resolved + if (m_forward_resolves_to) { + return; + } + + if (!looksResolvable(true)) { + return; + } + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + std::cout << "autoresolving " << TypeOrPyobj(this).name() << "\n"; + for (auto t: typesNeedingResolution) { + std::cout << " -> " << TypeOrPyobj(t).name() << "\n"; + if (t->isFunction()) { + std::cout << " " << (((Function*)t)->hasUnresolvedSymbols(false) ? "true":"false") << "\n"; + std::cout << " " << (((Function*)t)->hasUnresolvedSymbols(true) ? "true":"false") << "\n"; + } + } + + attemptToResolve(); + + + // if we're here, we didn't throw, so we must be resolved + // now walk each of our types and see if it wants to be + // installed somewhere specific. we can do this by looking + // for unstallable forwards and forcing them to resolve + for (auto t: typesNeedingResolution) { + if (t->isForward()) { + ((Forward*)t)->installDefinitionIfLambda(); + } + + if (t->isFunction()) { + ((Function*)t)->autoresolveGlobals(typesNeedingResolution); + } + } +} + PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { int num_args = 0; if (args) diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 59d54d258..66dbf0b3c 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -908,6 +908,14 @@ class Type { return m_forward_resolves_to; } + // attempt to autoresolve this type. This should + // always suceed and assumes we are holding the gil. + void tryToAutoresolve(); + + bool isRedundant() { + return m_is_redundant; + } + protected: Type(TypeCategory in_typeCategory) : m_typeCategory(in_typeCategory), @@ -973,10 +981,6 @@ class Type { m_is_redundant = true; } - bool isRedundant() { - return m_is_redundant; - } - void initializeFromConcrete(Type* forwardDefinitionOfSelf) { throw std::runtime_error("Type " + name() + " didn't define initializeFromConcrete"); } diff --git a/typed_python/TypeOrPyobj.cpp b/typed_python/TypeOrPyobj.cpp index 226f3ef93..127914936 100644 --- a/typed_python/TypeOrPyobj.cpp +++ b/typed_python/TypeOrPyobj.cpp @@ -92,7 +92,8 @@ TypeOrPyobj TypeOrPyobj::withoutIntern(PyObject* o) { std::string TypeOrPyobj::name() const { if (mType) { std::ostringstream s; - s << "name() + " of cat " + Type::categoryToString(mType->getTypeCategory()) + "@" << (void*)mType << ">"; + s << "<" << (mType->isForwardDefined() ? "Forward":"") + << "Type " << mType->name() + " of cat " + Type::categoryToString(mType->getTypeCategory()) + "@" << (void*)mType << ">"; return s.str(); } diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index f49e4a907..fcaf64eac 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -59,7 +59,9 @@ PyObject *MakeTupleOrListOfType(PyObject* nullValue, PyObject* args, bool isTupl return incref( (PyObject*)PyInstance::typeObj( - isTuple ? (TupleOrListOfType*)TupleOfType::Make(types[0]) : (TupleOrListOfType*)ListOfType::Make(types[0]) + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + isTuple ? (TupleOrListOfType*)TupleOfType::Make(types[0]) : (TupleOrListOfType*)ListOfType::Make(types[0]) + ) ) ); }); @@ -81,7 +83,12 @@ PyObject *MakePointerToType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } - return incref((PyObject*)PyInstance::typeObj(PointerTo::Make(t))); + return incref((PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + PointerTo::Make(t) + ) + ) + ); }); } @@ -102,7 +109,11 @@ PyObject *MakeRefToType(PyObject* nullValue, PyObject* args) { } return translateExceptionToPyObject([&]{ - return incref((PyObject*)PyInstance::typeObj(RefTo::Make(t))); + return incref((PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + RefTo::Make(t) + ) + )); }); }); } @@ -127,7 +138,12 @@ PyObject *MakeSubclassOfType(PyObject* nullValue, PyObject* args) { if ((t->isClass() && !((Class*)t)->isFinal()) || (t->isAlternative() && !((Class*)t)->isConcreteAlternative())) { return translateExceptionToPyObject([&]{ - return incref((PyObject*)PyInstance::typeObj(SubclassOfType::Make(t))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + SubclassOfType::Make(t) + ) + )); }); } @@ -151,7 +167,13 @@ PyObject *MakeTypedCellType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } - return incref((PyObject*)PyInstance::typeObj(TypedCellType::Make(t))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + TypedCellType::Make(t) + ) + ) + ); }); } @@ -174,7 +196,13 @@ PyObject *MakeTupleType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } - return incref((PyObject*)PyInstance::typeObj(Tuple::Make(types))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + Tuple::Make(types) + ) + ) + ); }); } @@ -195,8 +223,10 @@ PyObject *MakeConstDictType(PyObject* nullValue, PyObject* args) { } PyObject* typeObj = (PyObject*)PyInstance::typeObj( - ConstDictType::Make(types[0],types[1]) - ); + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + ConstDictType::Make(types[0],types[1]) + ) + ); return incref(typeObj); }); @@ -215,7 +245,11 @@ PyObject* MakeSetType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } SetType* setT = SetType::Make(t); - return incref((PyObject*)PyInstance::typeObj(setT)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(setT) + ) + ); }); } @@ -237,9 +271,11 @@ PyObject *MakeDictType(PyObject* nullValue, PyObject* args) { return incref( (PyObject*)PyInstance::typeObj( - DictType::Make(types[0],types[1]) + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + DictType::Make(types[0],types[1]) ) - ); + ) + ); }); } @@ -264,7 +300,11 @@ PyObject *MakeOneOfType(PyObject* nullValue, PyObject* args) { } } - PyObject* typeObj = (PyObject*)PyInstance::typeObj(OneOfType::Make(types)); + PyObject* typeObj = (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + OneOfType::Make(types) + ) + ); return incref(typeObj); }); @@ -317,7 +357,13 @@ PyObject *MakeNamedTupleType(PyObject* nullValue, PyObject* args, PyObject* kwar types.push_back(p.second); } - return incref((PyObject*)PyInstance::typeObj(NamedTuple::Make(types, names))); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + NamedTuple::Make(types, names) + ) + ) + ); }); } @@ -818,7 +864,11 @@ PyObject *MakeValueType(PyObject* nullValue, PyObject* args) { Type* type = PyInstance::tryUnwrapPyInstanceToValueType(arg, true); if (type) { - return incref((PyObject*)PyInstance::typeObj(type)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(type) + ) + ); } PyErr_Format(PyExc_TypeError, "Couldn't convert %S to a value", (PyObject*)arg); @@ -843,7 +893,11 @@ PyObject *MakeBoundMethodType(PyObject* nullValue, PyObject* args) { Type* resType = BoundMethod::Make(t0, PyUnicode_AsUTF8(a1)); - return incref((PyObject*)PyInstance::typeObj(resType)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(resType) + ) + ); } PyObject *MakeAlternativeMatcherType(PyObject* nullValue, PyObject* args) { @@ -867,7 +921,11 @@ PyObject *MakeAlternativeMatcherType(PyObject* nullValue, PyObject* args) { Type* resType = AlternativeMatcher::Make((Alternative*)t0); - return incref((PyObject*)PyInstance::typeObj(resType)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(resType) + ) + ); } PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { @@ -1102,7 +1160,11 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { ); } - return incref((PyObject*)PyInstance::typeObj(resType)); + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(resType) + ) + ); }); } @@ -1277,16 +1339,18 @@ PyObject *MakeClassType(PyObject* nullValue, PyObject* args) { return translateExceptionToPyObject([&]() { return incref( (PyObject*)PyInstance::typeObj( - Class::Make( - name, - baseClasses, - final == Py_True, - members, - memberFuncs, - staticFuncs, - propertyFuncs, - clsMembers, - classMethodFuncs + PyTemporaryReferenceTracer::autoresolveOnNextInstruction( + Class::Make( + name, + baseClasses, + final == Py_True, + members, + memberFuncs, + staticFuncs, + propertyFuncs, + clsMembers, + classMethodFuncs + ) ) ) ); @@ -3430,9 +3494,13 @@ PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwa std::sort(definitions.begin(), definitions.end()); } - return incref((PyObject*)PyInstance::typeObj( - ::Alternative::Make(name, moduleName, definitions, functions) - )); + Type* t = Alternative::Make(name, moduleName, definitions, functions); + + return incref( + (PyObject*)PyInstance::typeObj( + PyTemporaryReferenceTracer::autoresolveOnNextInstruction(t) + ) + ); } PyObject *getTypePointer(PyObject* nullValue, PyObject* args) { diff --git a/typed_python/python_ast.py b/typed_python/python_ast.py index c2d6b62f3..bb27bb33e 100644 --- a/typed_python/python_ast.py +++ b/typed_python/python_ast.py @@ -1014,16 +1014,6 @@ def convertPyAstToAlgebraic(tree, fname, keepLineInformation=True): try: return converter(**args) except Exception: - from typed_python._types import identityHash - print("BOO!") - print(type(args['annotation'])) - print(converter.ElementType.ElementTypes[1]) - print(converter.ElementType.ElementTypes[1].Types) - print(identityHash(converter.ElementType.ElementTypes[1]).hex()) - print(converter.ElementType.ElementTypes[1].Types[0]) - print(OneOf(Expr, None)) - print(OneOf(Expr, None) is converter.ElementType.ElementTypes[1]) - raise UserWarning( "Failed to construct %s from %s with arguments\n%s\n\n%s" % ( converter, diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 8276350bf..76c497cb7 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -63,6 +63,28 @@ def test_untyped_functions_are_not_forward(): assert not isForwardDefined(type(g)) +def test_autoresolve_forwards(): + A = Alternative("A", X={}, Y=dict(inst=lambda: B)) + B = Alternative("B", X={}, Y=dict(inst=lambda: A)) + + assert not isForwardDefined(A) + assert not isForwardDefined(B) + + +def test_autoresolve_forwards_3cycle(): + A = Alternative("A", X={}, Y=dict(inst=lambda: C)) + B = Alternative("B", X={}, Y=dict(inst=lambda: A)) + + assert isForwardDefined(A) + assert isForwardDefined(B) + + C = Alternative("C", X={}, Y=dict(inst=lambda: B)) + + assert not isForwardDefined(A) + assert not isForwardDefined(B) + assert not isForwardDefined(C) + + def test_invalid_forwards(): with pytest.raises(TypeError): Forward(lambda: 1) @@ -191,6 +213,31 @@ def test_recursive_definition_of_self(): assert F.__name__ == 'ListOf(^0)' +def test_make_recursive_list_of_autoresolve(): + def makeRecursiveListOf(T): + L = ListOf(OneOf(T, Forward(lambda: L))) + return L + + assert not isForwardDefined(makeRecursiveListOf(int)) + + assert makeRecursiveListOf(int) is makeRecursiveListOf(int) + assert makeRecursiveListOf(int) is not makeRecursiveListOf(str) + + +def test_autoresolve_nonstorage_forwards(): + print("START") + A = Alternative("A", X={}, fA=lambda: B) + print("A is defined") + + # this is failing because we are resolving fB without + # looking through to the full graph because we are using + # 'reachableTypes' in the function definition which is + # just wrong and incorrectly implemented... + B = Alternative("B", X={}, fB=lambda: A) + + assert not isForwardDefined(B) + assert not isForwardDefined(A) + def test_recursive_list_of_forward(): F = Forward() O = OneOf(None, F) @@ -587,3 +634,11 @@ class CChild(C_base_fwd): # part of the graph being new, part being old # 8. clean-up in the whole Instance/PyInstance layer - kind of nasty # 9. we should memoize instances of statless Function objects and instances of regular TP lists/tuple of, etc. + +# CHECK: +# OneOfs containing forwards that resolve to other oneofs changing the number +# of items - this needs to work +# MRTG should be completely ignoring Forwards - ideally they go away entirely +# - if they get leaked, that should be an error! +# how does this play with the ModuleRepresentation stuff? +# how does this play with TypeFunction? \ No newline at end of file From d6a941abd222f31f18511e49c061324c4f69f190 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 14 Jun 2023 15:57:33 +0000 Subject: [PATCH 30/83] Resolve some more bugs in forward type declaration. --- typed_python/ForwardType.cpp | 91 ++++++++++++++++++-- typed_python/ForwardType.hpp | 79 ++--------------- typed_python/FunctionType.hpp | 78 +++++++++++------ typed_python/PyInstance.cpp | 14 ++- typed_python/PyTemporaryReferenceTracer.cpp | 20 ++++- typed_python/Type.cpp | 13 ++- typed_python/compiler/python_ast_analysis.py | 4 + typed_python/compiler/python_ast_util.py | 18 +++- typed_python/internals.py | 5 +- typed_python/python_ast.py | 2 +- typed_python/type_construction_test.py | 88 ++++++++++++++----- typed_python/util.hpp | 9 ++ 12 files changed, 282 insertions(+), 139 deletions(-) diff --git a/typed_python/ForwardType.cpp b/typed_python/ForwardType.cpp index 22713728a..df4291fb6 100644 --- a/typed_python/ForwardType.cpp +++ b/typed_python/ForwardType.cpp @@ -17,6 +17,65 @@ #include "PyInstance.hpp" #include "ForwardType.hpp" +/* static */ +Forward* Forward::MakeFromFunction(PyObject* funcObj) { + if (!PyFunction_Check(funcObj)) { + throw std::runtime_error( + "Forwards can only be made from functions which return a single variable lookup." + ); + } + + PyCodeObject* code = (PyCodeObject*)PyFunction_GetCode(funcObj); + PyObject* closure = PyFunction_GetClosure(funcObj); + + if (!PyTuple_Check(code->co_names)) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names is not a tuple"); + } + + if (closure) { + PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); + + if (PyTuple_Size(code->co_names)) { + throw std::runtime_error("Corrupt Forward lookup function: can't have more than one name"); + } + + if (!PyTuple_Check(coFreevars)) { + throw std::runtime_error("Corrupt Forward lookup function: co_freevars was not a tuple"); + } + + if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { + throw std::runtime_error("Corrupt Forward lookup function: co_freevars not the same size as closure"); + } + + if (PyTuple_Size(coFreevars) != 1) { + throw std::runtime_error("Corrupt Forward lookup function: can't have more than 1 free variable"); + } + + PyObject* varname = PyTuple_GetItem(coFreevars, 0); + if (!PyUnicode_Check(varname)) { + throw std::runtime_error("Corrupt Forward lookup function: closure varnames need to be strings"); + } + + std::string name = PyUnicode_AsUTF8(varname); + + if (!PyCell_Check(PyTuple_GetItem(closure, 0))) { + throw std::runtime_error("Corrupt Forward lookup function: closure was not a cell"); + } + + return new Forward(name, PyTuple_GetItem(closure, 0)); + } + + if (PyTuple_Size(code->co_names) != 1) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); + } + + if (!PyUnicode_Check(PyTuple_GetItem(code->co_names, 0))) { + throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); + } + + std::string name = PyUnicode_AsUTF8(PyTuple_GetItem(code->co_names, 0)); + return new Forward(name, PyFunction_GetGlobals(funcObj)); +} Type* Forward::lambdaDefinition() { if (!mCellOrDict) { @@ -24,7 +83,13 @@ Type* Forward::lambdaDefinition() { } if (PyCell_Check(mCellOrDict) && PyCell_Get(mCellOrDict)) { - return PyInstance::unwrapTypeArgToTypePtr(PyCell_Get(mCellOrDict)); + Type* res = PyInstance::unwrapTypeArgToTypePtr(PyCell_Get(mCellOrDict)); + + if (res == this) { + return nullptr; + } + + return res; } if (PyDict_Check(mCellOrDict)) { @@ -34,12 +99,22 @@ Type* Forward::lambdaDefinition() { return nullptr; } - return PyInstance::unwrapTypeArgToTypePtr(item); + Type* res = PyInstance::unwrapTypeArgToTypePtr(item); + + if (res == this) { + return nullptr; + } + + return res; } return nullptr; } +bool Forward::lambdaDefinitionPopulated() { + return lambdaDefinition(); +} + void Forward::installDefinitionIfLambda() { if (!mCellOrDict) { return; @@ -57,11 +132,11 @@ void Forward::installDefinitionIfLambda() { if (cellContents) { // see if cellContents resolves - if (cellContents->isResolved() - && cellContents->forwardResolvesTo() + if (cellContents->isResolved() + && cellContents->forwardResolvesTo() == m_forward_resolves_to) { PyCell_Set( - mCellOrDict, + mCellOrDict, (PyObject*)PyInstance::typeObj( m_forward_resolves_to ) @@ -81,11 +156,11 @@ void Forward::installDefinitionIfLambda() { (PyTypeObject*)curContents ); - if (curContentsType && curContentsType->isResolved() && - curContentsType->forwardResolvesTo() + if (curContentsType && curContentsType->isResolved() && + curContentsType->forwardResolvesTo() == m_forward_resolves_to) { PyDict_SetItemString( - mCellOrDict, + mCellOrDict, m_name.c_str(), (PyObject*)PyInstance::typeObj( m_forward_resolves_to diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index a296b160e..bf1d38701 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -66,64 +66,7 @@ class Forward : public Type { return new Forward(name); } - static Forward* MakeFromFunction(PyObject* funcObj) { - if (!PyFunction_Check(funcObj)) { - throw std::runtime_error( - "Forwards can only be made from functions which return a single variable lookup." - ); - } - - PyCodeObject* code = (PyCodeObject*)PyFunction_GetCode(funcObj); - PyObject* closure = PyFunction_GetClosure(funcObj); - - if (!PyTuple_Check(code->co_names)) { - throw std::runtime_error("Corrupt Forward lookup function: Code object co_names is not a tuple"); - } - - if (closure) { - PyObjectStealer coFreevars(PyObject_GetAttrString(PyFunction_GetCode(funcObj), "co_freevars")); - - if (PyTuple_Size(code->co_names)) { - throw std::runtime_error("Corrupt Forward lookup function: can't have more than one name"); - } - - if (!PyTuple_Check(coFreevars)) { - throw std::runtime_error("Corrupt Forward lookup function: co_freevars was not a tuple"); - } - - if (PyTuple_Size(coFreevars) != PyTuple_Size(closure)) { - throw std::runtime_error("Corrupt Forward lookup function: co_freevars not the same size as closure"); - } - - if (PyTuple_Size(coFreevars) != 1) { - throw std::runtime_error("Corrupt Forward lookup function: can't have more than 1 free variable"); - } - - PyObject* varname = PyTuple_GetItem(coFreevars, 0); - if (!PyUnicode_Check(varname)) { - throw std::runtime_error("Corrupt Forward lookup function: closure varnames need to be strings"); - } - - std::string name = PyUnicode_AsUTF8(varname); - - if (!PyCell_Check(PyTuple_GetItem(closure, 0))) { - throw std::runtime_error("Corrupt Forward lookup function: closure was not a cell"); - } - - return new Forward(name, PyTuple_GetItem(closure, 0)); - } - - if (PyTuple_Size(code->co_names) != 1) { - throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); - } - - if (!PyUnicode_Check(PyTuple_GetItem(code->co_names, 0))) { - throw std::runtime_error("Corrupt Forward lookup function: Code object co_names refers to more than 1 name"); - } - - std::string name = PyUnicode_AsUTF8(PyTuple_GetItem(code->co_names, 0)); - return new Forward(name, PyFunction_GetGlobals(funcObj)); - } + static Forward* MakeFromFunction(PyObject* funcObj); Type* getTargetTransitive() { if (!mTarget) { @@ -164,6 +107,10 @@ class Forward : public Type { throw std::runtime_error("Forward is already resolved."); } + if (target == this) { + throw std::runtime_error("Forward can't be resolved to itself."); + } + mTarget = target; return target; @@ -199,21 +146,7 @@ class Forward : public Type { return mCellOrDict != nullptr; } - bool lambdaDefinitionPopulated() { - if (!mCellOrDict) { - return false; - } - - if (PyCell_Check(mCellOrDict)) { - return PyCell_Get(mCellOrDict) != nullptr; - } - - if (PyDict_Check(mCellOrDict)) { - return PyDict_GetItemString(mCellOrDict, m_name.c_str()); - } - - return false; - } + bool lambdaDefinitionPopulated(); Type* lambdaDefinition(); diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 976caa614..d4f9e83d1 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -778,18 +778,13 @@ class Function : public Type { return mArgs; } - template - void _visitReferencedTypes(const visitor_type& visitor) { - // we need to keep mArgs and mReturnType in sync with - // mAnnotations, so if we change one of our types we - // need to update the resulting dictionary as well. + void finalizeTypeConcrete() { + // ensure that the Function object we represent is in sync with + // our actual return and argument types. It would be better to not have + // any direct references to the python interpreter in this class and + // rebuild the concrete function object on demand later... if (mReturnType) { - Type* retType = mReturnType; - - visitor(mReturnType); - - if (mReturnType != retType - && mFunctionAnnotations + if (mFunctionAnnotations && PyDict_Check(mFunctionAnnotations) ) { PyDict_SetItemString( @@ -802,9 +797,7 @@ class Function : public Type { for (auto& a: mArgs) { Type* typeFilter = a.getTypeFilter(); - a._visitReferencedTypes(visitor); - - if (a.getTypeFilter() != typeFilter + if (typeFilter && mFunctionAnnotations && PyDict_Check(mFunctionAnnotations) ) { @@ -815,13 +808,53 @@ class Function : public Type { ); } } + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + // we need to keep mArgs and mReturnType in sync with + // mAnnotations, so if we change one of our types we + // need to update the resulting dictionary as well. + if (mReturnType) { + visitor(mReturnType); + } + for (auto& a: mArgs) { + a._visitReferencedTypes(visitor); + } for (auto& varnameAndBinding: mClosureBindings) { varnameAndBinding.second._visitReferencedTypes(visitor); } + if (mMethodOf) { visitor(mMethodOf); } + + for (auto nameAndGlobal: mFunctionGlobalsInCells) { + if (!PyCell_Check(nameAndGlobal.second)) { + throw std::runtime_error( + "A global in mFunctionGlobalsInCells is somehow not a cell" + ); + } + + PyObject* cellContents = PyCell_Get(nameAndGlobal.second); + + if (cellContents && PyType_Check(cellContents)) { + Type* t = PyInstance::extractTypeFrom(cellContents); + if (t) { + visitor(t); + } + } + } + + _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { + if (PyType_Check(val)) { + Type* t = PyInstance::extractTypeFrom(val); + if (t) { + visitor(t); + } + } + }); } template @@ -1239,25 +1272,18 @@ class Function : public Type { } void autoresolveGlobal( - std::string name, + std::string name, const std::set& resolvedForwards ) { - std::cout << "want to autoresolve " << name << "\n"; - auto it = mFunctionGlobalsInCells.find(name); if (it != mFunctionGlobalsInCells.end()) { - std::cout << "considering " << name << "\n"; if (PyCell_Check(it->second) && PyCell_GET(it->second)) { if (PyType_Check(PyCell_GET(it->second))) { Type* cellContents = PyInstance::extractTypeFrom( (PyTypeObject*)PyCell_Get(it->second) ); - std::cout << "look at " << TypeOrPyobj(cellContents).name() << "\n"; - if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - std::cout << "autoresvole " << name << " as cell\n"; - PyCell_Set( it->second, (PyObject*)PyInstance::typeObj( @@ -1278,8 +1304,6 @@ class Function : public Type { ); if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - std::cout << "autoresvole " << name << "\n"; - PyDict_SetItemString( mFunctionGlobals, name.c_str(), @@ -1721,6 +1745,12 @@ class Function : public Type { } } + void finalizeTypeConcrete() { + for (auto& o: mOverloads) { + o.finalizeTypeConcrete(); + } + } + std::string computeRecursiveNameConcrete(TypeStack& typeStack) { return mRootName; } diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 227162dc0..efd7175d7 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1720,6 +1720,17 @@ Type* PyInstance::tryUnwrapPyInstanceToType(PyObject* arg) { return possibleType; } + if (PyFunction_Check(arg)) { + try { + return Forward::MakeFromFunction(arg); + } catch(std::exception& e) { + return nullptr; + } catch(PythonExceptionSet& e) { + PyErr_Clear(); + return nullptr; + } + } + if (PyDict_Contains(nonTypesAcceptedAsTypes(), arg)) { PyObject* nonType = PyDict_GetItem(nonTypesAcceptedAsTypes(), arg); if (!nonType) { @@ -1788,7 +1799,8 @@ Type* PyInstance::unwrapTypeArgToTypePtr(PyObject* typearg) { try { return Forward::MakeFromFunction(typearg); } catch(std::exception& e) { - // do nothing + PyErr_Format(PyExc_TypeError, e.what()); + return NULL; } } diff --git a/typed_python/PyTemporaryReferenceTracer.cpp b/typed_python/PyTemporaryReferenceTracer.cpp index 0e93cd2f5..1d0524372 100644 --- a/typed_python/PyTemporaryReferenceTracer.cpp +++ b/typed_python/PyTemporaryReferenceTracer.cpp @@ -84,14 +84,14 @@ int PyTemporaryReferenceTracer::globalTraceFun(PyObject* dummyObj, PyFrameObject } else if (frameAction.action == TraceAction::Decref) { decref(frameAction.obj); - } + } else if (frameAction.action == TraceAction::Autoresolve) { PyErrorStasher stashCurrentException; // attempt to autoresolve the type. Note that if we // fail to do so, we just swallow the exception, which // is not great, but we can't be throwing exceptions - // in here... + // in here... try { frameAction.typ->tryToAutoresolve(); } catch(std::exception& e) { @@ -214,6 +214,22 @@ Type* PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* o) { PyThreadState *tstate = PyThreadState_GET(); PyFrameObject *f = tstate->frame; + // search upwards until we find a frame that's not running in typed_python.internals + auto isInternals = [&]() { + if (!PyUnicode_Check(f->f_code->co_filename)) { + return false; + } + + return endsWith(PyUnicode_AsUTF8(f->f_code->co_filename), "typed_python/internals.py"); + }; + + while (isInternals()) { + f = PyFrame_GetBack(f); + if (!f) { + throw std::runtime_error("Frame failed"); + } + } + if (f) { PyTemporaryReferenceTracer::autoresolveOnNextInstruction(o, f); } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index b56fe88db..7cebc9f0d 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -518,17 +518,14 @@ void Type::tryToAutoresolve() { std::set typesNeedingResolution; reachableUnresolvedTypes(this, typesNeedingResolution); - std::cout << "autoresolving " << TypeOrPyobj(this).name() << "\n"; - for (auto t: typesNeedingResolution) { - std::cout << " -> " << TypeOrPyobj(t).name() << "\n"; - if (t->isFunction()) { - std::cout << " " << (((Function*)t)->hasUnresolvedSymbols(false) ? "true":"false") << "\n"; - std::cout << " " << (((Function*)t)->hasUnresolvedSymbols(true) ? "true":"false") << "\n"; + if (::getenv("TP_AUTORESOLVE_VERBOSE") && ::getenv("TP_AUTORESOLVE_VERBOSE")[0]) { + std::cout << "try to autoresolve " << TypeOrPyobj(this).name() << ":\n"; + for (auto t: typesNeedingResolution) { + std::cout << " " << TypeOrPyobj(t).name() << "\n"; } } - - attemptToResolve(); + attemptToResolve(); // if we're here, we didn't throw, so we must be resolved // now walk each of our types and see if it wants to be diff --git a/typed_python/compiler/python_ast_analysis.py b/typed_python/compiler/python_ast_analysis.py index 589b1a519..a953b8459 100644 --- a/typed_python/compiler/python_ast_analysis.py +++ b/typed_python/compiler/python_ast_analysis.py @@ -233,7 +233,11 @@ def extractFunctionDefsInOrder(astNode): def visit(x): if isinstance(x, Statement): if x.matches.FunctionDef or x.matches.ClassDef or x.matches.AsyncFunctionDef: + if x.matches.FunctionDef: + visitPyAstChildren(x.returns, visit) + res.append(x) + return False if isinstance(x, Expr): diff --git a/typed_python/compiler/python_ast_util.py b/typed_python/compiler/python_ast_util.py index 50e085b5a..6049096dc 100644 --- a/typed_python/compiler/python_ast_util.py +++ b/typed_python/compiler/python_ast_util.py @@ -80,6 +80,9 @@ def pyAstForCode(codeObject): wholeFileAst = pyAstFromText(sourceText) + if codeObject.co_name == '': + return wholeFileAst + defs = functionDefsOrLambdaAtLineNumber(sourceText, wholeFileAst, codeObject.co_firstlineno) if len(defs) == 0: @@ -177,6 +180,7 @@ class AtLineNumberVisitor(ast.NodeVisitor): def __init__(self, lineNumber): self.funcDefSubnodesAtLineNumber = [] self.lambdaSubnodesAtLineNumber = [] + self.classDefSubnodesAtLineNumber = [] self.lineNumber = lineNumber def visit_FunctionDef(self, node): @@ -190,6 +194,17 @@ def visit_FunctionDef(self, node): ast.NodeVisitor.generic_visit(self, node) + def visit_ClassDef(self, node): + if node.lineno == self.lineNumber: + self.classDefSubnodesAtLineNumber.append(node) + else: + for d in node.decorator_list: + if d.lineno == self.lineNumber: + self.classDefSubnodesAtLineNumber.append(node) + break + + ast.NodeVisitor.generic_visit(self, node) + def visit_AsyncFunctionDef(self, node): if node.lineno == self.lineNumber: self.funcDefSubnodesAtLineNumber.append(node) @@ -208,5 +223,6 @@ def functionDefsOrLambdaAtLineNumber(sourceText, sourceAst, lineNumber): return ( visitor.funcDefSubnodesAtLineNumber + - visitor.lambdaSubnodesAtLineNumber + visitor.lambdaSubnodesAtLineNumber + + visitor.classDefSubnodesAtLineNumber ) diff --git a/typed_python/internals.py b/typed_python/internals.py index 1b33393a1..6ce9ef716 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -637,7 +637,7 @@ def extractCodeObjectNewStatementLineNumbers(codeObject): res = set() - if ast.matches.FunctionDef: + if ast.matches.FunctionDef or ast.matches.Module: res = extractLineNumbersWithStatements(ast.body) return res @@ -646,4 +646,7 @@ def extractCodeObjectNewStatementLineNumbers(codeObject): import traceback traceback.print_exc() + print("BAAD!") + print(codeObject) + return [] diff --git a/typed_python/python_ast.py b/typed_python/python_ast.py index bb27bb33e..8e40dfe3f 100644 --- a/typed_python/python_ast.py +++ b/typed_python/python_ast.py @@ -1255,7 +1255,7 @@ def cacheAstForCode(code, pyAst): codeConstants = [c for c in code.co_consts if isinstance(c, types.CodeType)] - if isinstance(pyAst, (Statement.FunctionDef, Expr.Lambda, Statement.ClassDef, Statement.AsyncFunctionDef)): + if isinstance(pyAst, (Statement.FunctionDef, Expr.Lambda, Statement.ClassDef, Statement.AsyncFunctionDef, Module)): funcDefs = extractFunctionDefsInOrder(pyAst.body) else: funcDefs = extractFunctionDefsInOrder(pyAst.generators) diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 76c497cb7..a60284dc0 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -118,6 +118,9 @@ def test_dual_recursive_types_direct_instantiation(): O1.define(ListOf(OneOf(int, O2))) O2.define(ListOf(OneOf(int, O1))) + assert isForwardDefined(O1) + assert isForwardDefined(O2) + O1 = resolveForwardDefinedType(O1) O2 = resolveForwardDefinedType(O2) @@ -129,15 +132,8 @@ def test_dual_recursive_types_direct_instantiation_with_alternatives(): O1.define(Alternative("O1", X={}, Y={'a': O2}, f=lambda self: O2)) O2.define(Alternative("O2", X={}, Y={'a': O1}, f=lambda self: O1)) - O1 = resolveForwardDefinedType(O1) - O2 = resolveForwardDefinedType(O2) - - -def test_dual_recursive_types_implicit_instantiation_with_alternatives(): - O1 = Alternative("O1", X={}, Y={'a': lambda: O2}, f=lambda self: O2) - - # this assignment triggers a resolution - O2 = Alternative("O2", X={}, Y={'a': lambda: O1}, f=lambda self: O1) + assert not isForwardDefined(O1) + assert not isForwardDefined(O2) def test_nonforward_definition(): @@ -225,19 +221,13 @@ def makeRecursiveListOf(T): def test_autoresolve_nonstorage_forwards(): - print("START") A = Alternative("A", X={}, fA=lambda: B) - print("A is defined") - - # this is failing because we are resolving fB without - # looking through to the full graph because we are using - # 'reachableTypes' in the function definition which is - # just wrong and incorrectly implemented... B = Alternative("B", X={}, fB=lambda: A) assert not isForwardDefined(B) assert not isForwardDefined(A) - + + def test_recursive_list_of_forward(): F = Forward() O = OneOf(None, F) @@ -255,6 +245,59 @@ def test_recursive_list_of_forward(): assert O_resolved.__name__ == 'OneOf(None, ListOf(^1))' +def test_define_class_holding_self(): + class C(Class): + anotherC = Member(OneOf(None, lambda: C)) + + def f(self) -> Forward(lambda: C): + return self + + assert not isForwardDefined(C) + + c = C() + assert type(c.f()) is C + + +def test_define_class_holding_self_defined_as_forward(): + C = Forward(lambda: C) + + assert isForwardDefined(C) + assert not typeLooksResolvable(C, True) + + class C(Class): + anotherC = Member(OneOf(None, C)) + + def f(self) -> C: + return self + + assert not isForwardDefined(C) + + c = C() + assert type(c.f()) is C + + +def test_two_mutually_recursive_classes(): + C1 = Forward(lambda: C1) + C2 = Forward(lambda: C2) + + class C1(Class): + holdsC2 = Member(OneOf(None, C2)) + + def f(self) -> C1: + return self + + assert isForwardDefined(C1) + + class C2(Class): + holdsC1 = Member(OneOf(None, C1)) + + def f(self) -> C2: + return self + + assert not isForwardDefined(C1) + assert not isForwardDefined(C2) + + def test_recursive_list_of_tuple_and_forward(): F = Forward() NT = NamedTuple(f=F) @@ -588,8 +631,11 @@ def makeClass(): assert A1_fwd is not A2_fwd # they have to point to the same concrete instance - assert A1_fwd.get() is A2_fwd.get() - assert A1_fwd.get() is A1 + assert isForwardDefined(A1_fwd.get()) + assert isForwardDefined(A2_fwd.get()) + + assert resolveForwardDefinedType(A1_fwd.get()) is A1 + assert resolveForwardDefinedType(A2_fwd.get()) is A1 @pytest.mark.skip(reason='TODO: make this work') @@ -638,7 +684,9 @@ class CChild(C_base_fwd): # CHECK: # OneOfs containing forwards that resolve to other oneofs changing the number # of items - this needs to work +# Forwards resolving to simple types like ints/floats # MRTG should be completely ignoring Forwards - ideally they go away entirely # - if they get leaked, that should be an error! # how does this play with the ModuleRepresentation stuff? -# how does this play with TypeFunction? \ No newline at end of file +# how does this play with TypeFunction? +# mutually recursive across modules? diff --git a/typed_python/util.hpp b/typed_python/util.hpp index 5f30cc8c0..f60e6a95c 100644 --- a/typed_python/util.hpp +++ b/typed_python/util.hpp @@ -430,6 +430,15 @@ inline bool startsWith(std::string name, std::string prefix) { } +inline bool endsWith(std::string name, std::string suffix) { + if (name.size() < suffix.size()) { + return false; + } + + return name.substr(name.size() - suffix.size(), suffix.size()) == suffix; +} + + class PyErrorStasher { public: PyErrorStasher() { From d19c87658c8550ffdb755118ecd686de4f162578 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 14 Jun 2023 21:03:07 +0000 Subject: [PATCH 31/83] Working autoresolution pathway. --- typed_python/BoundMethodType.hpp | 2 + typed_python/ConstDictType.hpp | 1 + typed_python/DictType.hpp | 1 + typed_python/FunctionType.hpp | 112 ++++++++------- typed_python/OneOfType.hpp | 1 + typed_python/PyFunctionInstance.cpp | 11 +- typed_python/PyFunctionInstance.hpp | 1 + typed_python/PyTemporaryReferenceTracer.cpp | 4 + typed_python/PyTemporaryReferenceTracer.hpp | 9 +- typed_python/SetType.hpp | 1 + typed_python/SubclassOfType.hpp | 1 + typed_python/TupleOrListOfType.hpp | 2 + typed_python/Type.cpp | 92 +++++++++++- typed_python/Type.hpp | 10 +- typed_python/TypedClosureBuilder.hpp | 2 +- typed_python/ValueType.cpp | 1 + typed_python/__init__.py | 2 + typed_python/_types.cpp | 77 +++++++++- typed_python/internals.py | 3 - typed_python/python_ast.py | 150 ++++++++++---------- typed_python/type_construction_test.py | 34 ++++- 21 files changed, 375 insertions(+), 142 deletions(-) diff --git a/typed_python/BoundMethodType.hpp b/typed_python/BoundMethodType.hpp index 9dbf95c48..d9db4c579 100644 --- a/typed_python/BoundMethodType.hpp +++ b/typed_python/BoundMethodType.hpp @@ -38,6 +38,8 @@ class BoundMethod : public Type { m_funcName = funcName; m_first_arg = inFirstArg; m_size = inFirstArg->bytecount(); + + recomputeName(); } const char* docConcrete() { diff --git a/typed_python/ConstDictType.hpp b/typed_python/ConstDictType.hpp index e02c8e3e4..c186de523 100644 --- a/typed_python/ConstDictType.hpp +++ b/typed_python/ConstDictType.hpp @@ -50,6 +50,7 @@ class ConstDictType : public Type { m_value(value) { m_is_forward_defined = true; + recomputeName(); } void initializeDuringDeserialization(Type* keyType, Type* valType) { diff --git a/typed_python/DictType.hpp b/typed_python/DictType.hpp index 341d53319..ac757a90b 100644 --- a/typed_python/DictType.hpp +++ b/typed_python/DictType.hpp @@ -43,6 +43,7 @@ class DictType : public Type { m_value(value) { m_is_forward_defined = true; + recomputeName(); } void initializeDuringDeserialization(Type* keyType, Type* valType) { diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index d4f9e83d1..06ed24614 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1214,18 +1214,16 @@ class Function : public Type { outAccesses.insert("__module_hash__"); } - static void extractNamesFromCode(PyCodeObject* code, std::set& outNames) { - iterate(code->co_names, [&](PyObject* o) { outNames.insert(o); }); - iterate(code->co_freevars, [&](PyObject* o) { outNames.insert(o); }); + void extractGlobalAccessesFromCodeIncludingCells( + std::set& outNames + ) const { + for (auto nameAndGlobal: mFunctionGlobalsInCells) { + outNames.insert(nameAndGlobal.first); + } - iterate(code->co_consts, [&](PyObject* o) { - if (PyCode_Check(o)) { - extractNamesFromCode((PyCodeObject*)o, outNames); - } - }); + outNames.insert("__module_hash__"); - static PyObject* moduleHashName = PyUnicode_FromString("__module_hash__"); - outNames.insert(moduleHashName); + extractGlobalAccessesFromCode((PyCodeObject*)mFunctionCode, outNames); } void setGlobals(PyObject* globals) { @@ -1255,17 +1253,23 @@ class Function : public Type { return true; } - if (mFunctionGlobals && PyDict_Check(mFunctionGlobals) - && PyDict_GetItemString(mFunctionGlobals, name.c_str())) { - if (insistResolved) { - PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - Type* t = PyInstance::extractTypeFrom(o); - if (t && t->isForwardDefined()) { - return true; + if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { + if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { + if (insistResolved) { + PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + Type* t = PyInstance::extractTypeFrom(o); + if (t && t->isForwardDefined()) { + return true; + } } + + return false; } - return false; + PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); + if (builtins && PyDict_GetItemString(builtins, name.c_str())) { + return false; + } } return true; @@ -1316,53 +1320,52 @@ class Function : public Type { } void autoresolveGlobals(const std::set& resolvedForwards) { - std::set allNames; - extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); - for (auto name: allNames) { - if (PyUnicode_Check(name)) { - std::string nameStr = PyUnicode_AsUTF8(name); - - if (nameStr != "__module_hash__") { - autoresolveGlobal(nameStr, resolvedForwards); - } + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__") { + autoresolveGlobal(nameStr, resolvedForwards); } } } bool hasUnresolvedSymbols(bool insistResolved) { - std::set allNames; - extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); - - for (auto name: allNames) { - if (PyUnicode_Check(name)) { - std::string nameStr = PyUnicode_AsUTF8(name); + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); - if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { - return true; - } + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { + return true; } } return false; } + std::string firstUnresolvedSymbol(bool insistResolved) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { + return nameStr; + } + } + + return ""; + } + PyObject* getUsedGlobals() const { // restrict the globals to contain only the values it references. PyObject* result = PyDict_New(); - std::set allNames; - extractNamesFromCode((PyCodeObject*)mFunctionCode, allNames); - std::set allNamesString; + extractGlobalAccessesFromCodeIncludingCells(allNamesString); - for (auto name: allNames) { - if (PyUnicode_Check(name)) { - std::string nameStr = PyUnicode_AsUTF8(name); - - if (mClosureBindings.find(nameStr) == mClosureBindings.end()) { - allNamesString.insert(nameStr); - } + for (auto nameStr: allNamesString) { + if (mClosureBindings.find(nameStr) == mClosureBindings.end()) { + throw std::runtime_error("Somehow we have a closure binding and a global"); } } @@ -1718,17 +1721,24 @@ class Function : public Type { } std::string nameWithModuleConcrete() { - if (mModulename.size() == 0) { - return mRootName; - } - - return mModulename + "." + mRootName; + return moduleNameConcrete() + "." + (mQualname.size() ? mQualname : mRootName); } // does this function have any of its globals that are not // resolved to actual values. if 'insistResolved' then we // also return 'true' if the symbols resolve to a forward defined // type + std::string firstUnresolvedSymbol(bool insistResolved) { + for (auto& o: mOverloads) { + std::string res = o.firstUnresolvedSymbol(insistResolved); + if (res.size()) { + return res; + } + } + + return ""; + } + bool hasUnresolvedSymbols(bool insistResolved) { for (auto& o: mOverloads) { if (o.hasUnresolvedSymbols(insistResolved)) { diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index 987c295d5..f4dc06b51 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -38,6 +38,7 @@ class OneOfType : public Type { m_types(types) { m_is_forward_defined = true; + recomputeName(); } const char* docConcrete() { diff --git a/typed_python/PyFunctionInstance.cpp b/typed_python/PyFunctionInstance.cpp index 860bdee37..3ca04ed12 100644 --- a/typed_python/PyFunctionInstance.cpp +++ b/typed_python/PyFunctionInstance.cpp @@ -934,7 +934,7 @@ PyObject* PyFunctionInstance::overload(PyObject* funcObj, PyObject* args, PyObje throw PythonExceptionSet(); } - argT = convertPythonObjectToFunctionType(name, arg, false, false); + argT = convertPythonObjectToFunctionType(name, nullptr, arg, false, false); } if (!argT) { @@ -1106,6 +1106,7 @@ PyObject* PyFunctionInstance::withOverloadVariableBindings(PyObject* cls, PyObje Function* PyFunctionInstance::convertPythonObjectToFunctionType( PyObject* name, + PyObject* classname, PyObject *funcObj, bool assumeClosuresGlobal, bool ignoreAnnotations @@ -1124,7 +1125,13 @@ Function* PyFunctionInstance::convertPythonObjectToFunctionType( PyObject* makeFunctionType = staticPythonInstance("typed_python.internals", "makeFunctionType"); - PyObjectStealer args(PyTuple_Pack(2, name, funcObj)); + PyObjectHolder args; + if (classname) { + args.steal(PyTuple_Pack(3, name, funcObj, classname)); + } else { + args.steal(PyTuple_Pack(2, name, funcObj)); + } + PyObjectStealer kwargs(PyDict_New()); if (assumeClosuresGlobal) { diff --git a/typed_python/PyFunctionInstance.hpp b/typed_python/PyFunctionInstance.hpp index a1aae8387..d7a9deab7 100644 --- a/typed_python/PyFunctionInstance.hpp +++ b/typed_python/PyFunctionInstance.hpp @@ -127,6 +127,7 @@ class PyFunctionInstance : public PyInstance { static Function* convertPythonObjectToFunctionType( PyObject* name, + PyObject* classname, PyObject *funcObj, bool assumeClosuresGlobal, // if true, then place closures in the function type itself // this is appropriate if this function is a method of a class diff --git a/typed_python/PyTemporaryReferenceTracer.cpp b/typed_python/PyTemporaryReferenceTracer.cpp index 1d0524372..573ac48c8 100644 --- a/typed_python/PyTemporaryReferenceTracer.cpp +++ b/typed_python/PyTemporaryReferenceTracer.cpp @@ -96,6 +96,9 @@ int PyTemporaryReferenceTracer::globalTraceFun(PyObject* dummyObj, PyFrameObject frameAction.typ->tryToAutoresolve(); } catch(std::exception& e) { // just swallow it + std::cout << "Warning: autoresolve on " + << frameAction.typ->name() << " failed: " << e.what() << "\n"; + } catch(PythonExceptionSet& e) { PyErr_Clear(); } @@ -232,6 +235,7 @@ Type* PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* o) { if (f) { PyTemporaryReferenceTracer::autoresolveOnNextInstruction(o, f); + o->registerContainingFrameForAutoresolve(f); } return o; diff --git a/typed_python/PyTemporaryReferenceTracer.hpp b/typed_python/PyTemporaryReferenceTracer.hpp index 589b8bdeb..9620d42cd 100644 --- a/typed_python/PyTemporaryReferenceTracer.hpp +++ b/typed_python/PyTemporaryReferenceTracer.hpp @@ -30,7 +30,8 @@ class PyTemporaryReferenceTracer { PyTemporaryReferenceTracer() : mostRecentEmptyFrame(nullptr), priorTraceFunc(nullptr), - priorTraceFuncArg(nullptr) + priorTraceFuncArg(nullptr), + autoresolutionEnabled(false) {} // perform an action on the first instruction where a @@ -60,6 +61,10 @@ class PyTemporaryReferenceTracer { int lineNumber; }; + void enableTypeAutoresolution(bool enabled) { + autoresolutionEnabled = enabled; + } + std::unordered_map > frameToActions; std::unordered_map > codeObjectToExpressionLines; @@ -69,6 +74,8 @@ class PyTemporaryReferenceTracer { Py_tracefunc priorTraceFunc; + bool autoresolutionEnabled; + PyObject* priorTraceFuncArg; bool isLineNewStatement(PyObject* code, int line); diff --git a/typed_python/SetType.hpp b/typed_python/SetType.hpp index 5e53e1a4d..a275f5ac5 100644 --- a/typed_python/SetType.hpp +++ b/typed_python/SetType.hpp @@ -37,6 +37,7 @@ class SetType : public Type { , m_key_type(eltype) { m_is_forward_defined = true; + recomputeName(); } const char* docConcrete() { diff --git a/typed_python/SubclassOfType.hpp b/typed_python/SubclassOfType.hpp index 917f0e05e..cc4b6a952 100644 --- a/typed_python/SubclassOfType.hpp +++ b/typed_python/SubclassOfType.hpp @@ -35,6 +35,7 @@ class SubclassOfType : public Type { m_subclassOf(subclassOf) { m_is_forward_defined = true; + recomputeName(); } const char* docConcrete() { diff --git a/typed_python/TupleOrListOfType.hpp b/typed_python/TupleOrListOfType.hpp index 63c717863..4a9514d06 100644 --- a/typed_python/TupleOrListOfType.hpp +++ b/typed_python/TupleOrListOfType.hpp @@ -48,6 +48,8 @@ class TupleOrListOfType : public Type { { m_is_forward_defined = true; m_is_default_constructible = true; + + recomputeName(); } void initializeDuringDeserialization(Type* elementType) { diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 7cebc9f0d..22e698572 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -309,6 +309,42 @@ bool Type::looksResolvable(bool unambiguously) { return true; } + +bool Type::assertResolvable(bool unambiguously) { + if (!m_is_forward_defined) { + return true; + } + + if (m_forward_resolves_to) { + return true; + } + + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + throw std::runtime_error("Forward " + t->name() + " has no definition."); + } + } + + if (unambiguously) { + for (auto t: typesNeedingResolution) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols(false)) { + throw std::runtime_error( + "Function " + t->nameWithModule() + " has unresolved symbol '" + + ((Function*)t)->firstUnresolvedSymbol(false) + "'" + ); + } + } + } + + return true; +} + // try to resolve this forward type. If we can't, we'll throw an exception. On exit, // we will have thrown, or m_forward_resolves_to will be populated. void Type::attemptToResolve() { @@ -527,10 +563,10 @@ void Type::tryToAutoresolve() { attemptToResolve(); - // if we're here, we didn't throw, so we must be resolved - // now walk each of our types and see if it wants to be - // installed somewhere specific. we can do this by looking - // for unstallable forwards and forcing them to resolve + for (auto t: typesNeedingResolution) { + t->attemptAutoresolveWrite(); + } + for (auto t: typesNeedingResolution) { if (t->isForward()) { ((Forward*)t)->installDefinitionIfLambda(); @@ -542,6 +578,54 @@ void Type::tryToAutoresolve() { } } +void Type::attemptAutoresolveWrite() { + if (!m_is_forward_defined || !m_forward_resolves_to) { + return; + } + + if (!mAutoresolveFrameOwners.size()) { + return; + } + + long updateCount = 0; + + for (auto f: mAutoresolveFrameOwners) { + // force the locals to be populated with a dict so we can muck with them + PyFrame_FastToLocals((PyFrameObject*)f); + + PyObject* locals = ((PyFrameObject*)f)->f_locals; + + if (locals && PyDict_Check(locals)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + + PyObject* ownTypeObj = (PyObject*)PyInstance::typeObj(this); + PyObject* resolvedTypeObj = (PyObject*)PyInstance::typeObj(m_forward_resolves_to); + + bool updated = false; + while (PyDict_Next(locals, &pos, &key, &value)) { + if (value == ownTypeObj) { + PyDict_SetItem(locals, key, resolvedTypeObj); + updated = true; + } + } + + if (updated) { + updateCount += 1; + PyFrame_LocalsToFast((PyFrameObject*)f, 0); + } + } + + decref(f); + } + + mAutoresolveFrameOwners.clear(); + + if (::getenv("TP_AUTORESOLVE_VERBOSE") && ::getenv("TP_AUTORESOLVE_VERBOSE")[0]) { + std::cout << "Autoresolve wrote " << name() << " into " << updateCount << " stackframe slots\n"; + } +} + PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { int num_args = 0; if (args) diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 66dbf0b3c..92549fe01 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -895,6 +895,7 @@ class Type { } bool looksResolvable(bool unambiguously); + bool assertResolvable(bool unambiguously); Type* forwardResolvesTo() { if (!m_is_forward_defined) { @@ -916,6 +917,13 @@ class Type { return m_is_redundant; } + // indicate that this frame may contain a reference to this object in its locals + void registerContainingFrameForAutoresolve(PyFrameObject* frame) { + mAutoresolveFrameOwners.push_back(incref((PyObject*)frame)); + } + + void attemptAutoresolveWrite(); + protected: Type(TypeCategory in_typeCategory) : m_typeCategory(in_typeCategory), @@ -947,7 +955,7 @@ class Type { Type* m_base; - enum BinaryCompatibilityCategory { Incompatible, Checking, Compatible }; + std::vector mAutoresolveFrameOwners; // a sha-hash that uniquely identifies this type. If this value is // the same for two types, then they should be indistinguishable except diff --git a/typed_python/TypedClosureBuilder.hpp b/typed_python/TypedClosureBuilder.hpp index 240defc78..ddc76cdd8 100644 --- a/typed_python/TypedClosureBuilder.hpp +++ b/typed_python/TypedClosureBuilder.hpp @@ -87,7 +87,7 @@ class TypedClosureBuilder { if (PyFunction_Check(o)) { PyObjectStealer name(PyObject_GetAttrString(o, "__name__")); - Function* funcType = PyFunctionInstance::convertPythonObjectToFunctionType(name, o, false, true); + Function* funcType = PyFunctionInstance::convertPythonObjectToFunctionType(name, nullptr, o, false, true); if (!funcType) { throw PythonExceptionSet(); diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 34b115ffd..2d04abb60 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -41,6 +41,7 @@ Value::Value(const Instance& instance) : } m_is_forward_defined = true; + recomputeName(); } Type* Value::Make(Instance i) { diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 4b2ec7c64..fe00b4efc 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -61,6 +61,8 @@ import typed_python._types as _types import threading +_types._enableTypeAutoresolution(True) + # in the c module, these are functions, but because they're not parametrized, # we want them to be actual values. Int8 = _types.Int8() diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index fcaf64eac..9a1d46ced 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2584,6 +2584,45 @@ PyObject *isForwardDefined(PyObject* nullValue, PyObject* args) { }); } +PyObject *isResolved(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "isResolved takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); + + return translateExceptionToPyObject([&]() { + // Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->isResolved() ? Py_True : Py_False); + // } + + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + throw std::runtime_error("first argument to 'isResolved' must be a type object"); + } + + return incref(t->isResolved() ? Py_True : Py_False); + }); +} + +PyObject *enableTypeAutoresolution(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + int enabled = 1; + + static const char *kwlist[] = {"enabled", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|p", (char**)kwlist, &enabled)) { + throw PythonExceptionSet(); + } + + PyTemporaryReferenceTracer::globalTracer.enableTypeAutoresolution(enabled); + + return incref(Py_None); + }); +} + PyObject *typeLooksResolvable(PyObject* nullValue, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { PyObject* type; @@ -2611,6 +2650,33 @@ PyObject *typeLooksResolvable(PyObject* nullValue, PyObject* args, PyObject* kwa }); } +PyObject *assertTypeResolvable(PyObject* nullValue, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* type; + int unambiguously = 0; + + static const char *kwlist[] = {"type", "unambiguously", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|p", (char**)kwlist, &type, &unambiguously)) { + throw PythonExceptionSet(); + } + + // see first if its an unclosured function object + // Type* typeOfArg = PyInstance::extractTypeFrom(type->ob_type, false); + // if (typeOfArg && typeOfArg->isFunction()) { + // return incref(typeOfArg->looksResolvable(unambiguously) ? Py_True : Py_False); + // } + + Type* t = PyInstance::unwrapTypeArgToTypePtr(type); + + if (!t) { + throw std::runtime_error("type must be a TP type instance"); + } + + return incref(t->assertResolvable(unambiguously) ? Py_True : Py_False); + }); +} + PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { PyErr_SetString(PyExc_TypeError, "resolveForwardDefinedType takes 1 positional argument"); @@ -3448,7 +3514,13 @@ PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwa std::string fieldName(PyUnicode_AsUTF8(key)); if (PyFunction_Check(value)) { - functions[fieldName] = PyFunctionInstance::convertPythonObjectToFunctionType(key, value, true, false); + functions[fieldName] = PyFunctionInstance::convertPythonObjectToFunctionType( + key, + PyTuple_GetItem(args, 0), + value, + true, + false + ); if (functions[fieldName] == nullptr) { //error code is already set @@ -3567,8 +3639,11 @@ static PyMethodDef module_methods[] = { {"buildPyFunctionObject", (PyCFunction)buildPyFunctionObject, METH_VARARGS | METH_KEYWORDS, NULL}, {"isPOD", (PyCFunction)isPOD, METH_VARARGS, NULL}, {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, + {"isResolved", (PyCFunction)isResolved, METH_VARARGS, NULL}, + {"_enableTypeAutoresolution", (PyCFunction)enableTypeAutoresolution, METH_VARARGS, NULL}, {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, {"typeLooksResolvable", (PyCFunction)typeLooksResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, + {"assertTypeResolvable", (PyCFunction)assertTypeResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, {"bytecount", (PyCFunction)bytecount, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroup", (PyCFunction)recursiveTypeGroup, METH_VARARGS | METH_KEYWORDS, NULL}, {"recursiveTypeGroupRepr", (PyCFunction)recursiveTypeGroupRepr, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/typed_python/internals.py b/typed_python/internals.py index 6ce9ef716..1124281cb 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -646,7 +646,4 @@ def extractCodeObjectNewStatementLineNumbers(codeObject): import traceback traceback.print_exc() - print("BAAD!") - print(codeObject) - return [] diff --git a/typed_python/python_ast.py b/typed_python/python_ast.py index 8e40dfe3f..453091345 100644 --- a/typed_python/python_ast.py +++ b/typed_python/python_ast.py @@ -24,29 +24,30 @@ import types import traceback -from typed_python._types import Forward, Alternative, TupleOf, OneOf, resolveForwardDefinedType - +from typed_python._types import ( + Forward, Alternative, TupleOf, OneOf, resolveForwardDefinedType +) # forward declarations. -Module = Forward("Module") -Statement = Forward("Statement") -Expr = Forward("Expr") -Arg = Forward("Arg") -NumericConstant = Forward("NumericConstant") -ExprContext = Forward("ExprContext") -BooleanOp = Forward("BooleanOp") -BinaryOp = Forward("BinaryOp") -UnaryOp = Forward("UnaryOp") -ComparisonOp = Forward("ComparisonOp") -Comprehension = Forward("Comprehension") -ExceptionHandler = Forward("ExceptionHandler") -Arguments = Forward("Arguments") -Keyword = Forward("Keyword") -Alias = Forward("Alias") -WithItem = Forward("WithItem") -TypeIgnore = Forward("TypeIgnore") - -Module.define(Alternative( +Module = Forward(lambda: Module) +Statement = Forward(lambda: Statement) +Expr = Forward(lambda: Expr) +Arg = Forward(lambda: Arg) +NumericConstant = Forward(lambda: NumericConstant) +ExprContext = Forward(lambda: ExprContext) +BooleanOp = Forward(lambda: BooleanOp) +BinaryOp = Forward(lambda: BinaryOp) +UnaryOp = Forward(lambda: UnaryOp) +ComparisonOp = Forward(lambda: ComparisonOp) +Comprehension = Forward(lambda: Comprehension) +ExceptionHandler = Forward(lambda: ExceptionHandler) +Arguments = Forward(lambda: Arguments) +Keyword = Forward(lambda: Keyword) +Alias = Forward(lambda: Alias) +WithItem = Forward(lambda: WithItem) +TypeIgnore = Forward(lambda: TypeIgnore) + +Module = Alternative( "Module", Module={ "body": TupleOf(Statement), @@ -55,12 +56,12 @@ Expression={'body': Expr}, Interactive={'body': TupleOf(Statement)}, Suite={"body": TupleOf(Statement)} -)) +) -TypeIgnore.define(Alternative( +TypeIgnore = Alternative( "TypeIgnore", Item={'lineno': int, 'tag': str} -)) +) def statementStrLines(self): @@ -161,7 +162,7 @@ def StatementStr(self): return "\n".join(list(statementStrLines(self))) -Statement.define(Alternative( +Statement = Alternative( "Statement", FunctionDef={ "name": str, @@ -356,7 +357,7 @@ def StatementStr(self): 'filename': str }, __str__=StatementStr -)) +) def ExpressionStr(self): @@ -475,7 +476,7 @@ def ExpressionStr(self): return str(type(self)) -Expr.define(Alternative( +Expr = Alternative( "Expr", BoolOp={ "op": BooleanOp, @@ -682,9 +683,9 @@ def ExpressionStr(self): 'filename': str }, __str__=ExpressionStr -)) +) -NumericConstant.define(Alternative( +NumericConstant = Alternative( "NumericConstant", Int={"value": int}, Long={"value": str}, @@ -700,9 +701,9 @@ def ExpressionStr(self): ) else "None" if self.matches.None_ else f"{self.real} + {self.imag}j" if self.matches.Complex else "Unknown" ) -)) +) -ExprContext.define(Alternative( +ExprContext = Alternative( "ExprContext", Load={}, Store={}, @@ -710,15 +711,15 @@ def ExpressionStr(self): AugLoad={}, AugStore={}, Param={} -)) +) -BooleanOp.define(Alternative( +BooleanOp = Alternative( "BooleanOp", And={}, Or={} -)) +) -BinaryOp.define(Alternative( +BinaryOp = Alternative( "BinaryOp", Add={}, Sub={}, @@ -733,17 +734,17 @@ def ExpressionStr(self): BitAnd={}, FloorDiv={}, MatMult={} -)) +) -UnaryOp.define(Alternative( +UnaryOp = Alternative( "UnaryOp", Invert={}, Not={}, UAdd={}, USub={} -)) +) -ComparisonOp.define(Alternative( +ComparisonOp = Alternative( "ComparisonOp", Eq={}, NotEq={}, @@ -755,9 +756,9 @@ def ExpressionStr(self): IsNot={}, In={}, NotIn={} -)) +) -Comprehension.define(Alternative( +Comprehension = Alternative( "Comprehension", Item={ "target": Expr, @@ -765,9 +766,9 @@ def ExpressionStr(self): "ifs": TupleOf(Expr), "is_async": bool } -)) +) -ExceptionHandler.define(Alternative( +ExceptionHandler = Alternative( "ExceptionHandler", Item={ "type": OneOf(Expr, None), @@ -777,9 +778,9 @@ def ExpressionStr(self): 'col_offset': int, 'filename': str } -)) +) -Arguments.define(Alternative( +Arguments = Alternative( "Arguments", Item={ **({'posonlyargs': TupleOf(Arg)} if sys.version_info.minor >= 8 else {}), @@ -800,9 +801,9 @@ def ExpressionStr(self): + ([self.vararg.arg] if self.vararg else []) + [a.arg for a in self.kwonlyargs] + ([self.kwarg.arg] if self.kwarg else []) -)) +) -Arg.define(Alternative( +Arg = Alternative( "Arg", Item={ 'arg': str, @@ -811,18 +812,18 @@ def ExpressionStr(self): 'col_offset': int, 'filename': str } -)) +) -Keyword.define(Alternative( +Keyword = Alternative( "Keyword", Item={ "arg": OneOf(None, str), "value": Expr, **({'line_number': int, 'col_offset': int, 'filename': str} if sys.version_info.minor >= 9 else {}) } -)) +) -Alias.define(Alternative( +Alias = Alternative( "Alias", Item={ "name": str, @@ -833,15 +834,37 @@ def ExpressionStr(self): 'filename': str } if sys.version_info.minor >= 10 else {}) } -)) +) -WithItem.define(Alternative( +WithItem = Alternative( "WithItem", Item={ "context_expr": Expr, "optional_vars": OneOf(Expr, None), } -)) +) + + +# because this module is defined inside of typed_python, the autoresolver will not be +# enabled yet. +Module = resolveForwardDefinedType(Module) +Statement = resolveForwardDefinedType(Statement) +Expr = resolveForwardDefinedType(Expr) +Arg = resolveForwardDefinedType(Arg) +NumericConstant = resolveForwardDefinedType(NumericConstant) +ExprContext = resolveForwardDefinedType(ExprContext) +BooleanOp = resolveForwardDefinedType(BooleanOp) +BinaryOp = resolveForwardDefinedType(BinaryOp) +UnaryOp = resolveForwardDefinedType(UnaryOp) +ComparisonOp = resolveForwardDefinedType(ComparisonOp) +Comprehension = resolveForwardDefinedType(Comprehension) +ExceptionHandler = resolveForwardDefinedType(ExceptionHandler) +Arguments = resolveForwardDefinedType(Arguments) +Keyword = resolveForwardDefinedType(Keyword) +Alias = resolveForwardDefinedType(Alias) +WithItem = resolveForwardDefinedType(WithItem) +TypeIgnore = resolveForwardDefinedType(TypeIgnore) + numericConverters = { int: lambda x: NumericConstant.Int(value=x), @@ -1386,25 +1409,6 @@ def evaluateFunctionDefWithLocalsInCells(pyAst, globals, locals, stripAnnotation return inner -Module = resolveForwardDefinedType(Module) -Statement = resolveForwardDefinedType(Statement) -Expr = resolveForwardDefinedType(Expr) -Arg = resolveForwardDefinedType(Arg) -NumericConstant = resolveForwardDefinedType(NumericConstant) -ExprContext = resolveForwardDefinedType(ExprContext) -BooleanOp = resolveForwardDefinedType(BooleanOp) -BinaryOp = resolveForwardDefinedType(BinaryOp) -UnaryOp = resolveForwardDefinedType(UnaryOp) -ComparisonOp = resolveForwardDefinedType(ComparisonOp) -Comprehension = resolveForwardDefinedType(Comprehension) -ExceptionHandler = resolveForwardDefinedType(ExceptionHandler) -Arguments = resolveForwardDefinedType(Arguments) -Keyword = resolveForwardDefinedType(Keyword) -Alias = resolveForwardDefinedType(Alias) -WithItem = resolveForwardDefinedType(WithItem) -TypeIgnore = resolveForwardDefinedType(TypeIgnore) - - # map Python AST types to our syntax-tree types converters = { ast.Module: Module.Module, diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index a60284dc0..e7ff1ee3e 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, Final, typeLooksResolvable + Class, Member, ConstDict, TypedCell, typeLooksResolvable ) from typed_python.test_util import CodeEvaluator @@ -33,7 +33,6 @@ def g(): def test_type_looks_resolvable_alternative(): - # note that these types are not autoresolvable because of their names A = Alternative("A_", X={}, f=lambda self: B) assert typeLooksResolvable(A, unambiguously=False) @@ -53,6 +52,31 @@ def test_type_looks_resolvable_alternative(): assert B().g() is A +def test_autoresolve_unnamed_type_with_named_forward(): + # this defines 'F' as a forward. When 'F' gets assigned to something other than a forward + # the type will become resolvable. + F = Forward(lambda: F) + + # this defines 'G' - there's no reference in the type system to it + G = OneOf(None, F) + + # this allows 'F' and 'G' to be resolved. + F = ListOf(G) + + # the system was able to autoresolve both F and G because they were assigned + assert not isForwardDefined(F) + assert not isForwardDefined(G) + + +def test_autoresolve_unnamed_type_with_anonymous_forward(): + G = OneOf(None, lambda: F) + F = ListOf(G) + + # the system was able to autoresolve both F and G because they were assigned + assert not isForwardDefined(F) + assert not isForwardDefined(G) + + def test_untyped_functions_are_not_forward(): # this type is not forward defined because it should be holding 'g' in the closure of # 'f' itself @@ -169,9 +193,6 @@ def test_forward_definition(): assert isForwardDefined(Alternative("A", x={'f': F})) assert isForwardDefined(Alternative("A", x={'f': F}).x) - func = Function(lambda x: x, F) - assert isForwardDefined(type(func)) - def test_forward_one_of(): F = Forward() @@ -420,6 +441,7 @@ def makeA(): makeA().C(x=10) +@pytest.mark.skip(reason='what to do with forwards functions?') def test_cannot_call_forward_function_instances(): t = Forward("T") @@ -433,6 +455,7 @@ def f(x: t): f(10) +@pytest.mark.skip(reason='what to do with forwards functions?') def test_function_types_coalesc(): def makeFNonforward(T): @Function @@ -457,6 +480,7 @@ def f(x: t): assert makeFforward(int) is makeFforward(int) +@pytest.mark.skip(reason='what to do with forwards functions?') def test_instances_of_forward_function_are_forward(): T = Forward("T") From bad9e0a798640c08c83e91bbca95d7a7dcc27eb4 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 14 Jun 2023 22:52:51 +0000 Subject: [PATCH 32/83] Fix more bugs. --- typed_python/FunctionType.hpp | 9 ++- typed_python/HeldClassType.hpp | 10 ++-- typed_python/PyAlternativeInstance.cpp | 29 ++++++++- typed_python/PyTemporaryReferenceTracer.cpp | 8 +++ ...onSerializationContext_deserialization.cpp | 2 +- typed_python/Type.cpp | 45 ++++++++++++++ typed_python/Type.hpp | 9 +-- typed_python/__init__.py | 44 +++++++------- typed_python/compiler/python_ast_analysis.py | 4 +- .../compiler/type_wrappers/range_wrapper.py | 13 ++-- typed_python/internals.py | 12 +++- typed_python/test_util.py | 2 +- typed_python/type_construction_test.py | 52 +++++++++++++++- typed_python/types_serialization_test.py | 25 ++++---- typed_python/types_test.py | 60 ++++++++++++++----- 15 files changed, 245 insertions(+), 79 deletions(-) diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 06ed24614..05d47d4f4 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1364,8 +1364,11 @@ class Function : public Type { extractGlobalAccessesFromCodeIncludingCells(allNamesString); for (auto nameStr: allNamesString) { - if (mClosureBindings.find(nameStr) == mClosureBindings.end()) { - throw std::runtime_error("Somehow we have a closure binding and a global"); + if (nameStr != "__module_hash__" && mClosureBindings.find(nameStr) == mClosureBindings.end()) { + throw std::runtime_error( + "Somehow we have both a closure binding and a global for " + + nameStr + ); } } @@ -2125,7 +2128,7 @@ class Function : public Type { // //Function types can be instantiated even if forwards are not resolved // //in their annotations. - // void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { + // void assertForwardsResolvedSufficientlyToInstantiateConcrete() { // mClosureType->assertForwardsResolvedSufficientlyToInstantiate(); // } diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index b3b8d6569..6c24d797e 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -414,6 +414,8 @@ class HeldClass : public Type { for (auto& nameAndF: m_own_staticFunctions) { nameAndF.second = nameAndF.second->withMethodOf(this); } + + initializeMRO(); } void initializeDuringDeserialization( @@ -962,10 +964,6 @@ class HeldClass : public Type { } void initializeMRO() { - if (m_is_forward_defined) { - throw std::runtime_error("Makes no sense to initializeMRO on a forward"); - } - if (m_mro.size()) { return; } @@ -982,6 +980,10 @@ class HeldClass : public Type { throw std::runtime_error("Somehow " + m_name + " doesn't have itself as MRO 0"); } + if (m_is_forward_defined) { + return; + } + // build our own method resolution table directly from our parents. mergeOwnFunctionsIntoInheritanceTree(); diff --git a/typed_python/PyAlternativeInstance.cpp b/typed_python/PyAlternativeInstance.cpp index d4c68d6eb..dd4483577 100644 --- a/typed_python/PyAlternativeInstance.cpp +++ b/typed_python/PyAlternativeInstance.cpp @@ -422,9 +422,11 @@ std::pair PyConcreteAlternativeInstance::callMethod(const char* } auto res = PyFunctionInstance::tryToCallAnyOverload(method, nullptr, nullptr, targetArgTuple, nullptr); + if (res.first) { return res; } + PyErr_Format( PyExc_TypeError, "'%s.%s' cannot find a valid overload with these arguments", @@ -459,6 +461,10 @@ Py_ssize_t PyConcreteAlternativeInstance::mp_and_sq_length_concrete() { if (!p.first) { return -1; } + if (!p.second) { + return -1; + } + if (!PyLong_Check(p.second)) { return -1; } @@ -475,11 +481,28 @@ PyObject* PyConcreteAlternativeInstance::mp_subscript_concrete(PyObject* item) { } int PyConcreteAlternativeInstance::mp_ass_subscript_concrete(PyObject* item, PyObject* v) { - auto p = callMethod("__setitem__", item, v); - if (!p.first) { - PyErr_Format(PyExc_TypeError, "__setitem__ not defined for type %s", type()->name().c_str()); + std::pair res; + + if (!v) { + res = callMethod("__delitem__", item); + } else { + res = callMethod("__setitem__", item, v); + } + + if (!res.first) { + if (v) { + PyErr_Format(PyExc_TypeError, "__setitem__ not defined for type %s", type()->name().c_str()); + } else { + PyErr_Format(PyExc_TypeError, "__delitem__ not defined for type %s", type()->name().c_str()); + } + return -1; } + + if (!res.second) { + return -1; + } + return 0; } diff --git a/typed_python/PyTemporaryReferenceTracer.cpp b/typed_python/PyTemporaryReferenceTracer.cpp index 573ac48c8..2afb86c96 100644 --- a/typed_python/PyTemporaryReferenceTracer.cpp +++ b/typed_python/PyTemporaryReferenceTracer.cpp @@ -184,6 +184,10 @@ void PyTemporaryReferenceTracer::keepaliveForCurrentInstruction(PyObject* o, PyF } void PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* t, PyFrameObject* f) { + if (!globalTracer.autoresolutionEnabled) { + return; + } + // mark that we're going to trace globalTracer.frameToActions[f].push_back( FrameAction( @@ -210,6 +214,10 @@ void PyTemporaryReferenceTracer::traceObject(PyObject* o) { } Type* PyTemporaryReferenceTracer::autoresolveOnNextInstruction(Type* o) { + if (!globalTracer.autoresolutionEnabled) { + return o; + } + if (!o->isForwardDefined() || o->isResolved()) { return o; } diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index fc771bae6..a764a619e 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1624,7 +1624,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( } else if (fieldNumber == 8) { assertWireTypesEqual(wireType, WireType::VARINT); classIsFinal = b.readUnsignedVarint(); - } else if (fieldNumber == 9) { + } else if (fieldNumber == 10) { types.push_back(deserializeNativeType(b, wireType)); } } else diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 22e698572..f02d2c266 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -309,6 +309,51 @@ bool Type::looksResolvable(bool unambiguously) { return true; } +void Type::assertForwardsResolvedSufficientlyToInstantiateConcrete() { + if (!m_is_forward_defined) { + return; + } + + if (m_forward_resolves_to) { + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be instantiated. However, " + + "has already been resolved, so it needs only to be replaced with a concrete " + + "type by calling resolveForwardDeclaredType and it can be used." + ); + } + + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); + + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be autoresolved because " + + "it refers to forward " + t->nameWithModule() + " which has no definition." + ); + } + } + + for (auto t: typesNeedingResolution) { + if (t->isFunction() && ((Function*)t)->hasUnresolvedSymbols(false)) { + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be autoresolved because " + + "it refers to forward " + t->nameWithModule() + " which has a reference to " + + "symbol '" + ((Function*)t)->firstUnresolvedSymbol(false) + "' which is " + + "unresolved." + ); + } + } + + throw std::logic_error( + nameWithModule() + " is forward-declared and cannot be instantiated. However, " + + "it is resolvable, so it needs only to be replaced with a concrete " + + "type by calling resolveForwardDeclaredType and it can be used." + ); +} bool Type::assertResolvable(bool unambiguously) { if (!m_is_forward_defined) { diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 92549fe01..e37cf3d3f 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -701,14 +701,7 @@ class Type { }); } - void assertForwardsResolvedSufficientlyToInstantiateConcrete() const { - if (m_is_forward_defined) { - throw std::logic_error( - "Type " + m_name + " of cat " - + this->getTypeCategoryString() + " is a forward" - ); - } - } + void assertForwardsResolvedSufficientlyToInstantiateConcrete(); template void _visitReferencedTypes(const visitor_type& v) {} diff --git a/typed_python/__init__.py b/typed_python/__init__.py index fe00b4efc..6aad85446 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -40,7 +40,6 @@ from typed_python.module import Module from typed_python.type_function import TypeFunction from typed_python.hash import sha_hash -from typed_python.SerializationContext import SerializationContext from typed_python.compiler.typeof import TypeOf from typed_python._types import ( Forward, TupleOf, ListOf, Tuple, NamedTuple, OneOf, ConstDict, SubclassOf, @@ -61,8 +60,6 @@ import typed_python._types as _types import threading -_types._enableTypeAutoresolution(True) - # in the c module, these are functions, but because they're not parametrized, # we want them to be actual values. Int8 = _types.Int8() @@ -76,27 +73,30 @@ EmbeddedMessage = _types.EmbeddedMessage() PyCell = _types.PyCell() -# from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa +_types._enableTypeAutoresolution(True) + +from typed_python.SerializationContext import SerializationContext # noqa +from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa -# from typed_python.generator import Generator # noqa +from typed_python.generator import Generator # noqa -# # this has to come at the end to break import cyclic -# from typed_python.lib.map import map # noqa -# from typed_python.lib.pmap import pmap # noqa -# from typed_python.lib.reduce import reduce # noqa +# this has to come at the end to break import cyclic +from typed_python.lib.map import map # noqa +from typed_python.lib.pmap import pmap # noqa +from typed_python.lib.reduce import reduce # noqa -# _types.initializeGlobalStatics() +_types.initializeGlobalStatics() -# # start a background thread to release the GIL for us. Instead of immediately releasing the GIL, -# # we prefer to release it a short time after our C code no longer needs it, in case it -# # reacquires it immediately, which is very slow. -# # this is needed primarily because we often can't tell whether we're about to enter code -# # that acquires and releases the GIL very frequently (say, in a tight loop), which has -# # a huge performance penalty (a few thousand context switches per second!) -# # instead, we have a thread that checks in the background whether any thread wants us -# # to release, and if so, swap it out. This can yield a 10-50x performance improvement -# # when we're acquiring and releasing frequently. -# gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) -# gilReleaseThreadLoop.start() +# start a background thread to release the GIL for us. Instead of immediately releasing the GIL, +# we prefer to release it a short time after our C code no longer needs it, in case it +# reacquires it immediately, which is very slow. +# this is needed primarily because we often can't tell whether we're about to enter code +# that acquires and releases the GIL very frequently (say, in a tight loop), which has +# a huge performance penalty (a few thousand context switches per second!) +# instead, we have a thread that checks in the background whether any thread wants us +# to release, and if so, swap it out. This can yield a 10-50x performance improvement +# when we're acquiring and releasing frequently. +gilReleaseThreadLoop = threading.Thread(target=_types.gilReleaseThreadLoop, daemon=True) +gilReleaseThreadLoop.start() -# _types.setGilReleaseThreadLoopSleepMicroseconds(500) +_types.setGilReleaseThreadLoopSleepMicroseconds(500) diff --git a/typed_python/compiler/python_ast_analysis.py b/typed_python/compiler/python_ast_analysis.py index a953b8459..eff99e596 100644 --- a/typed_python/compiler/python_ast_analysis.py +++ b/typed_python/compiler/python_ast_analysis.py @@ -205,11 +205,11 @@ def extractLineNumbersWithStatements(astNode): def visit(x): if isinstance(x, Statement): + res.add(x.line_number) + if x.matches.FunctionDef or x.matches.ClassDef or x.matches.AsyncFunctionDef: return False - res.add(x.line_number) - if isinstance(x, Expr): if ( x.matches.Lambda diff --git a/typed_python/compiler/type_wrappers/range_wrapper.py b/typed_python/compiler/type_wrappers/range_wrapper.py index 7bc862f5f..502ddaf54 100644 --- a/typed_python/compiler/type_wrappers/range_wrapper.py +++ b/typed_python/compiler/type_wrappers/range_wrapper.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typed_python import Class, Final, PointerTo, Held, Member, pointerTo +from typed_python import Class, Final, PointerTo, Held, Member, pointerTo, resolveForwardDefinedType class Range(Class, Final, __name__='range'): @@ -32,9 +32,6 @@ def __call__(self, start, stop, step): # noqa return RangeCls(start=start, stop=stop, step=step) -range = Range() - - @Held class RangeCls(Class, Final): start = Member(int, nonempty=True) @@ -93,3 +90,11 @@ def __fastnext__(self) -> PointerTo(int): return startPtr return PointerTo(int)() + + +RangeCls = resolveForwardDefinedType(RangeCls) +RangeIterator = resolveForwardDefinedType(RangeIterator) +Range = resolveForwardDefinedType(Range) + + +range = Range() diff --git a/typed_python/internals.py b/typed_python/internals.py index 1124281cb..47b5bad7e 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -520,8 +520,16 @@ def addArg(self, name, defaultVal, typeFilter, isStarArg, isKwarg): def __str__(self): if self.methodOf: - return "FunctionOverload(%s, returns %s, %s)" % (self.methodOf.Class.__name__, self.returnType, self.args) - return "FunctionOverload(returns %s, %s)" % (self.returnType, self.args) + return "FunctionOverload(%s, returns %s, %s)" % ( + self.methodOf.Class.__name__, + self.returnType if self.returnType is not None else 'anything', + self.args + ) + + return "FunctionOverload(returns %s, %s)" % ( + self.returnType if self.returnType is not None else 'anything', + self.args + ) def _installNativePointer(self, fp, returnType, argumentTypes): typed_python._types.installNativeFunctionPointer( diff --git a/typed_python/test_util.py b/typed_python/test_util.py index 08410d063..351c45642 100644 --- a/typed_python/test_util.py +++ b/typed_python/test_util.py @@ -23,7 +23,7 @@ import os from typed_python import sha_hash -# from typed_python import Entrypoint, SerializationContext +from typed_python import Entrypoint, SerializationContext def currentMemUsageMb(residentOnly=True): diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index e7ff1ee3e..dd2c1bdaa 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, typeLooksResolvable + Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held ) from typed_python.test_util import CodeEvaluator @@ -68,6 +68,56 @@ def test_autoresolve_unnamed_type_with_named_forward(): assert not isForwardDefined(G) +def test_held_of_forward_class(): + class C(Class): + def f(self): + return H + + assert isForwardDefined(C) + + @Held + class H(Class): + def f(self): + return C + + assert not isForwardDefined(H) + assert not isForwardDefined(C) + + +def test_autoresolve_on_decorator(): + B = Forward(lambda: B) + + class B(Class): + def blah(self) -> B: + return self + + @Function + def callIt(x: B): + pass + + assert not isForwardDefined(B) + assert not isForwardDefined(type(callIt)) + + +def test_autoresolve_forwards_with_nonforwards(): + class C(Class): + def f(self): + return T + + assert isForwardDefined(C) + + with pytest.raises(TypeError): + C() + + T = int + + assert typeLooksResolvable(C, unambiguously=True) + + C = resolveForwardDefinedType(C) + + C() + + def test_autoresolve_unnamed_type_with_anonymous_forward(): G = OneOf(None, lambda: F) F = ListOf(G) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 9cf6bd052..1570b178f 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2500,9 +2500,8 @@ def test_identity_hash_of_lambda_doesnt_change_serialization(self): assert ser1 == ser2 def test_call_method_dispatch_on_two_versions_of_same_class_with_recursion(self): - Base = Forward("Base") + Base = Forward(lambda: Base) - @Base.define class Base(Class): def blah(self) -> Base: return self @@ -2514,8 +2513,6 @@ def f(self, x) -> int: def callF(x: Base): return x.f(10) - compilerHash(Base) - def deserializeAndCall(): class Child(Base, Final): def f(self, x) -> int: @@ -2525,20 +2522,24 @@ def f(self, x) -> int: return SerializationContext().serialize(Child()) - childBytes = callFunctionInFreshProcess(deserializeAndCall, ()) + SerializationContext().deserialize( + SerializationContext().serialize(deserializeAndCall) + ) - child1 = SerializationContext().deserialize(childBytes) - child2 = SerializationContext().deserialize(childBytes) + # childBytes = callFunctionInFreshProcess(deserializeAndCall, ()) - assert type(child1) is type(child2) + # child1 = SerializationContext().deserialize(childBytes) + # child2 = SerializationContext().deserialize(childBytes) + + # assert type(child1) is type(child2) - childVersionOfBase = type(child1).BaseClasses[0] + # childVersionOfBase = type(child1).BaseClasses[0] - assert typesAreEquivalent(Base, childVersionOfBase) + # assert typesAreEquivalent(Base, childVersionOfBase) - assert Base in type(child1).BaseClasses + # assert Base in type(child1).BaseClasses - assert callF(child1) == callF(child2) + # assert callF(child1) == callF(child2) @pytest.mark.skip( reason="serialization differences on 3.8 we need to investigate" diff --git a/typed_python/types_test.py b/typed_python/types_test.py index c3dcc3d56..a015382e7 100644 --- a/typed_python/types_test.py +++ b/typed_python/types_test.py @@ -33,8 +33,7 @@ ConstDict, Alternative, serialize, deserialize, Class, Function, Forward, Set, PointerTo, Entrypoint, - Final, - resolveForwardDefinedType + Final ) from typed_python.type_promotion import ( computeArithmeticBinaryResultType, floatness, bitness, isSignedInt @@ -44,9 +43,7 @@ AnAlternative = Alternative("AnAlternative", X={'x': int}) -AForwardAlternative = Forward() -AForwardAlternative.define(Alternative("AForwardAlternative", Y={}, X={'x': AForwardAlternative})) -AForwardAlternative = resolveForwardDefinedType(AForwardAlternative) +AForwardAlternative = Alternative("AForwardAlternative", Y={}, X={'x': lambda: AForwardAlternative}) def typeFor(t): @@ -1162,6 +1159,9 @@ def A_delattr(s, n): __delattr__=A_delattr, ) + from typed_python import resolveForwardDefinedType + A = resolveForwardDefinedType(A) + self.assertEqual(A.a().__bool__(), False) self.assertEqual(bool(A.a()), False) self.assertEqual(A.b().__bool__(), True) @@ -1244,26 +1244,54 @@ def A_delattr(s, n): d = dir(A.a()) self.assertGreater(len(d), 50) - A2_items = dict() + def test_alternative_magic_methods_2(self): + A2_attrs = {"q": "value-q", "z": "value-z"} + + def A2_getattr(s, n): + if n not in A2_attrs: + raise AttributeError(f"no attribute {n}") + return A2_attrs[n] + + def A2_setattr(s, n, v): + A2_attrs[n] = v + + def A2_delattr(s, n): + A2_attrs.pop(n, None) + + A2_items = {'blah': 'blah'} def A2_setitem(self, i, v): A2_items[i] = v - return 0 - A2 = Alternative("A2", a={'a': int}, b={'b': str}, - __getattribute__=A_getattr, - __setattr__=A_setattr, - __delattr__=A_delattr, - __dir__=lambda self: list(A_attrs.keys()), - __getitem__=lambda self, i: A2_items.get(i, i), - __setitem__=A2_setitem - ) + def A2_delitem(self, i): + A2_items.pop(i) + + A2 = Alternative( + "A2", + a={'a': int}, + b={'b': str}, + __getattribute__=A2_getattr, + __setattr__=A2_setattr, + __delattr__=A2_delattr, + __dir__=lambda self: list(A2_attrs.keys()), + __getitem__=lambda self, i: A2_items.get(i, i), + __setitem__=A2_setitem, + __delitem__=A2_delitem + ) + + assert 'blah' in A2_items + del A2.b()['blah'] + assert 'blah' not in A2_items + self.assertEqual(A2.b().q, "value-q") + A2.b().q = "changedvalue for q" self.assertEqual(A2.b().q, "changedvalue for q") A2.a().Name = "can change Name" self.assertEqual(A2.b().Name, "can change Name") - self.assertEqual(dir(A2.b()), ["Name", "q"]) + self.assertEqual(dir(A2.b()), ["Name", "q", "z"]) + self.assertEqual(A2.b()[123], 123) + A2.b()[123] = 7 self.assertEqual(A2.b()[123], 7) From 7b55b4911c3d1cac4ecc455112465befc6d69a7f Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 14 Jun 2023 22:54:37 +0000 Subject: [PATCH 33/83] More --- typed_python/types_serialization_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 1570b178f..5fedc5f37 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2526,20 +2526,20 @@ def f(self, x) -> int: SerializationContext().serialize(deserializeAndCall) ) - # childBytes = callFunctionInFreshProcess(deserializeAndCall, ()) + childBytes = callFunctionInFreshProcess(deserializeAndCall, ()) - # child1 = SerializationContext().deserialize(childBytes) - # child2 = SerializationContext().deserialize(childBytes) + child1 = SerializationContext().deserialize(childBytes) + child2 = SerializationContext().deserialize(childBytes) - # assert type(child1) is type(child2) + assert type(child1) is type(child2) - # childVersionOfBase = type(child1).BaseClasses[0] + childVersionOfBase = type(child1).BaseClasses[0] - # assert typesAreEquivalent(Base, childVersionOfBase) + assert typesAreEquivalent(Base, childVersionOfBase) - # assert Base in type(child1).BaseClasses + assert Base in type(child1).BaseClasses - # assert callF(child1) == callF(child2) + assert callF(child1) == callF(child2) @pytest.mark.skip( reason="serialization differences on 3.8 we need to investigate" From 05bf4987ece695e01d15dc8c244c901cf81592ca Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 15 Jun 2023 02:49:44 +0000 Subject: [PATCH 34/83] The problem is that when we first walk the type graph to internalize the object, the cell we're modifying points to something different than what we end up with after we've done the type finalization. To solve this, i think we need for each type graph to identify the cell storage / globals dicts it can use to see the rest of the graph and make those first-class citizens that are considered "part" of the object graph (in the sense that they'll get memoized, serialized, and 'resolved' the same way a type reference in another type would. --- repro.py | 37 +++++++++++ typed_python/FunctionType.hpp | 2 +- typed_python/HeldClassType.hpp | 2 + typed_python/MutuallyRecursiveTypeGroup.cpp | 11 +++- typed_python/PyInstance.cpp | 4 +- typed_python/PythonSerializationContext.hpp | 2 +- ...onSerializationContext_deserialization.cpp | 64 +++++++++++-------- ...thonSerializationContext_serialization.cpp | 7 +- typed_python/Type.cpp | 14 ++++ typed_python/Type.hpp | 19 +++++- typed_python/types_serialization_test.py | 15 ++--- 11 files changed, 132 insertions(+), 45 deletions(-) create mode 100644 repro.py diff --git a/repro.py b/repro.py new file mode 100644 index 000000000..52f365078 --- /dev/null +++ b/repro.py @@ -0,0 +1,37 @@ +import sys + +from typed_python import Class, SerializationContext, Held, isForwardDefined +from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr + + +def writer(): + @Held + class C(Class): + def f(self): + return C + + assert not isForwardDefined(C) + + print("group is:") + print(typeWalkRecord(type(C.f).overloads[0].funcGlobalsInCells['C'], 'identity')) + print(typeWalkRecord(type(C.f).overloads[0].funcGlobalsInCells['C'], 'compiler')) + + # with open("a.dat", "wb") as f: + # f.write(SerializationContext().serialize(C)) + + +def reader(): + with open("a.dat", "rb") as f: + C = SerializationContext().deserialize(f.read()) + + print(C) + #assert C().f() is C + +if sys.argv[1:] == ['r']: + reader() +else: + writer() + + + + diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 05d47d4f4..5431e99c9 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1364,7 +1364,7 @@ class Function : public Type { extractGlobalAccessesFromCodeIncludingCells(allNamesString); for (auto nameStr: allNamesString) { - if (nameStr != "__module_hash__" && mClosureBindings.find(nameStr) == mClosureBindings.end()) { + if (nameStr != "__module_hash__" && mClosureBindings.find(nameStr) != mClosureBindings.end()) { throw std::runtime_error( "Somehow we have both a closure binding and a global for " + nameStr diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 6c24d797e..25ea7edeb 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -434,6 +434,7 @@ class HeldClass : public Type { m_bases = bases; m_is_final = isFinal; m_own_members = members; + m_own_memberFunctions = memberFunctions; m_own_staticFunctions = staticFunctions; m_own_propertyFunctions = propertyFunctions; m_own_classMembers = classMembers; @@ -466,6 +467,7 @@ class HeldClass : public Type { void initializeFromConcrete(Type* forwardDef) { HeldClass* fwdCls = (HeldClass*)forwardDef; + m_name = fwdCls->m_name; m_classType = fwdCls->m_classType; m_bases = fwdCls->m_bases; m_is_final = fwdCls->m_is_final; diff --git a/typed_python/MutuallyRecursiveTypeGroup.cpp b/typed_python/MutuallyRecursiveTypeGroup.cpp index c27fcbb6a..f0d9b5297 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.cpp +++ b/typed_python/MutuallyRecursiveTypeGroup.cpp @@ -101,6 +101,8 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { + "\n\nand hashed it, we ended up with this group\n\n" + canonical->repr() + "\n\nwhich has a different number of elements" + + "\n\nmVisibilityType = " + visibilityTypeToStr(mVisibilityType) + + "\ncanonical->nmVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) ); } @@ -115,6 +117,8 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { + "\n\nand hashed it, we ended up with this group\n\n" + canonical->repr() + "\n\nwhich has a different hash!" + + "\n\nmVisibilityType = " + visibilityTypeToStr(mVisibilityType) + + "\ncanonical->nmVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) ); } @@ -169,7 +173,11 @@ void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { if (typeAndOrder.first.type() && mVisibilityType == VisibilityType::Identity) { Type* t = typeAndOrder.first.type(); - t->setRecursiveTypeGroup(this, typeAndOrder.second); + t->setRecursiveTypeGroup( + this, + typeAndOrder.second, + ShaHash(2) + this->hash() + ShaHash(typeAndOrder.second) + ); } } } @@ -814,7 +822,6 @@ void MutuallyRecursiveTypeGroup::ensureRecursiveTypeGroup(TypeOrPyobj root, Visi static thread_local int count = 0; count++; if (count > 1) { - asm("int3"); throw std::runtime_error( "There should be only one group algo running at once. Somehow, " "our reference to " + root.name() + " wasn't captured correctly." diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index efd7175d7..e373edcbb 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1258,7 +1258,9 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { categoryToPyString(inType->getTypeCategory()) ); - mirrorTypeInformationIntoPyType(inType, &types[inType]->typeObj); + if (!inType->isActivelyBeingDeserialized()) { + mirrorTypeInformationIntoPyType(inType, &types[inType]->typeObj); + } return (PyTypeObject*)types[inType]; } diff --git a/typed_python/PythonSerializationContext.hpp b/typed_python/PythonSerializationContext.hpp index 9578835bc..a0ababe4b 100644 --- a/typed_python/PythonSerializationContext.hpp +++ b/typed_python/PythonSerializationContext.hpp @@ -163,7 +163,7 @@ class PythonSerializationContext : public SerializationContext { void deserializeClassClassMemberDict(std::map& dict, DeserializationBuffer& b, int wireType) const; - static Type* constructBlankInstanceOfType(Type::TypeCategory typeCat); + static Type* constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat); void deserializeNativeTypeIntoBlank(DeserializationBuffer& b, size_t wireType, Type* blankShell, std::string inName) const; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index a764a619e..378da3ca8 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -574,75 +574,80 @@ PyObject* prepareBlankInstanceOfType(MutuallyRecursiveTypeGroup* outGroup, PyTyp } /* static */ -Type* PythonSerializationContext::constructBlankInstanceOfType(Type::TypeCategory typeCat) { +Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat) { + Type* result = nullptr; + if (typeCat == Type::TypeCategory::catValue) { - return new Value(); + result = new Value(); } if (typeCat == Type::TypeCategory::catOneOf) { - return new OneOfType(); + result = new OneOfType(); } if (typeCat == Type::TypeCategory::catTupleOf) { - return new TupleOfType(); + result = new TupleOfType(); } if (typeCat == Type::TypeCategory::catPointerTo) { - return new PointerTo(); + result = new PointerTo(); } if (typeCat == Type::TypeCategory::catListOf) { - return new ListOfType(); + result = new ListOfType(); } if (typeCat == Type::TypeCategory::catNamedTuple) { - return new NamedTuple(); + result = new NamedTuple(); } if (typeCat == Type::TypeCategory::catTuple) { - return new Tuple(); + result = new Tuple(); } if (typeCat == Type::TypeCategory::catDict) { - return new DictType(); + result = new DictType(); } if (typeCat == Type::TypeCategory::catConstDict) { - return new ConstDictType(); + result = new ConstDictType(); } if (typeCat == Type::TypeCategory::catAlternative) { - return new Alternative(); + result = new Alternative(); } if (typeCat == Type::TypeCategory::catConcreteAlternative) { - return new ConcreteAlternative(); + result = new ConcreteAlternative(); } if (typeCat == Type::TypeCategory::catPythonObjectOfType) { - return new PythonObjectOfType(); + result = new PythonObjectOfType(); } if (typeCat == Type::TypeCategory::catBoundMethod) { - return new BoundMethod(); + result = new BoundMethod(); } if (typeCat == Type::TypeCategory::catAlternativeMatcher) { - return new AlternativeMatcher(); + result = new AlternativeMatcher(); } if (typeCat == Type::TypeCategory::catClass) { - return new Class(); + result = new Class(); } if (typeCat == Type::TypeCategory::catHeldClass) { - return new HeldClass(); + result = new HeldClass(); } if (typeCat == Type::TypeCategory::catFunction) { - return new Function(); + result = new Function(); } if (typeCat == Type::TypeCategory::catForward) { throw std::runtime_error("Can't deserialize actual forwards!"); } if (typeCat == Type::TypeCategory::catSet) { - return new SetType(); + result = new SetType(); } if (typeCat == Type::TypeCategory::catTypedCell) { - return new TypedCellType(); + result = new TypedCellType(); } if (typeCat == Type::TypeCategory::catRefTo) { - return new RefTo(); + result = new RefTo(); } if (typeCat == Type::TypeCategory::catSubclassOf) { - return new SubclassOfType(); + result = new SubclassOfType(); } - + if (result) { + result->markActivelyBeingDeserialized(); + return result; + } if (typeCat == Type::TypeCategory::catPyCell || typeCat == Type::TypeCategory::catEmbeddedMessage @@ -762,7 +767,7 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur auto typeCat = Type::TypeCategory(b.readUnsignedVarint()); if (actuallyBuildGroup) { - Type* type = constructBlankInstanceOfType(typeCat); + Type* type = constructBlankSubclassOfTypeCategory(typeCat); outGroup->setIndexToObject(indexInGroup, type); indicesOfNativeTypes[indexInGroup] = type; } else { @@ -1118,7 +1123,7 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } } else if (fieldNumber == 6) { - // field 5 indicates that this is a mutually recursive type group + // field indicates that this is a mutually recursive type group // identified by a perfect factory PyObjectStealer factoryAndArgs(deserializePythonObject(b, wireType)); @@ -1213,6 +1218,13 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur ixAndType.second->finalizeType(); } + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->typeFinishedBeingDeserialized(); + } + + std::cout << "finalizing group:\n"; + std::cout << outGroup->repr() << "\n"; + outGroup->finalizeDeserializerGroup(); } @@ -1741,7 +1753,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( classClassMembersRaw[nameAndObj.first] = incref((PyObject*)nameAndObj.second); } - if (!blankShell->isForward()) { + if (!blankShell->isHeldClass()) { throw std::runtime_error("Shell is not a HeldClass"); } diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 969b610f2..52330d805 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -20,7 +20,7 @@ #include "MutuallyRecursiveTypeGroup.hpp" #include "ScopedIndenter.hpp" -#define TP_VERBOSE_SERIALIZE 0 +#define TP_VERBOSE_SERIALIZE 1 // virtual // serialize a python object. This function should work in any context. @@ -425,7 +425,8 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec # if TP_VERBOSE_SERIALIZE ScopedIndenter s; - std::cout << s.prefix() << "Serializing fresh MRTG: " << group->hash().digestAsHexString() << "\n"; + std::cout << s.prefix() << "Serializing fresh MRTG: " << group->hash().digestAsHexString() + << " with " << group->getIndexToObject().size() << " items" << "\n"; # endif // group hash if (shouldSerializeHashSequence()) { @@ -994,7 +995,7 @@ void PythonSerializationContext::serializeNativeTypeInner( b.writeUnsignedVarintObject(8, cls->isFinal() ? 1 : 0); - serializeNativeType(cls->getClassType(), b, 9); + serializeNativeType(cls->getClassType(), b, 10); } else { throw std::runtime_error( "Can't serialize native type " + nativeType->name() diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index f02d2c266..96ce4c909 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -671,6 +671,20 @@ void Type::attemptAutoresolveWrite() { } } +void Type::typeFinishedBeingDeserialized() { + if (m_is_being_deserialized) { + // during deserialization we have to defer anything that actually + // looks hard at the type and copies data into the PyTypeObject + // because its not ready yet. This allows us to refer to the object + // without it being finished. + PyTypeObject* typeObj = PyInstance::typeObj(this); + + PyInstance::mirrorTypeInformationIntoPyType(this, typeObj); + + m_is_being_deserialized = false; + } +} + PyObject* getOrSetTypeResolver(PyObject* module, PyObject* args) { int num_args = 0; if (args) diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index e37cf3d3f..51772f311 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -863,12 +863,13 @@ class Type { return mIdentityHash; } - void setRecursiveTypeGroup(MutuallyRecursiveTypeGroup* group, int32_t index) { + void setRecursiveTypeGroup(MutuallyRecursiveTypeGroup* group, int32_t index, ShaHash hash) { if (group->visibilityType() != VisibilityType::Identity) { throw std::runtime_error("A Type's MutuallyRecursiveTypeGroup needs to be an Identity group"); } mTypeGroup = group; mRecursiveTypeGroupIndex = index; + mIdentityHash = hash; } int64_t getRecursiveTypeGroupIndex() const { @@ -917,6 +918,16 @@ class Type { void attemptAutoresolveWrite(); + void markActivelyBeingDeserialized() { + m_is_being_deserialized = true; + } + + bool isActivelyBeingDeserialized() { + return m_is_being_deserialized; + } + + void typeFinishedBeingDeserialized(); + protected: Type(TypeCategory in_typeCategory) : m_typeCategory(in_typeCategory), @@ -929,11 +940,15 @@ class Type { mRecursiveTypeGroupIndex(-1), m_is_forward_defined(false), m_forward_resolves_to(nullptr), - m_is_redundant(false) + m_is_redundant(false), + m_is_being_deserialized(false) {} TypeCategory m_typeCategory; + // this class is actively being deserialized. + bool m_is_being_deserialized; + size_t m_size; bool m_is_default_constructible; diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 5fedc5f37..55b166e56 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2514,17 +2514,14 @@ def callF(x: Base): return x.f(10) def deserializeAndCall(): - class Child(Base, Final): - def f(self, x) -> int: - return -1 + # class Child(Base, Final): + # def f(self, x) -> int: + # return -1 - assert isinstance(Child(), Base) + # assert isinstance(Child(), Base) - return SerializationContext().serialize(Child()) - - SerializationContext().deserialize( - SerializationContext().serialize(deserializeAndCall) - ) + return SerializationContext().serialize(Base) + #return SerializationContext().serialize(Child()) childBytes = callFunctionInFreshProcess(deserializeAndCall, ()) From 6b6bcd1bb494ecc59400d8a930cf7ff5a9e777c7 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 16 Jun 2023 15:31:21 +0000 Subject: [PATCH 35/83] Refactor FunctionOverload into its own file. --- typed_python/ClosureVariableBinding.cpp | 87 + typed_python/ClosureVariableBinding.hpp | 362 ++++ typed_python/CompiledSpecialization.hpp | 58 + typed_python/CompilerVisibleObjectVisitor.hpp | 2 +- typed_python/FunctionArg.hpp | 174 ++ typed_python/FunctionCallArgMapping.hpp | 42 +- ...{FunctionType.cpp => FunctionOverload.cpp} | 90 +- typed_python/FunctionOverload.hpp | 1128 +++++++++++ typed_python/FunctionType.hpp | 1658 +---------------- .../ModuleRepresentationCopyContext.hpp | 8 +- typed_python/PyFunctionInstance.cpp | 18 +- typed_python/PyFunctionInstance.hpp | 4 +- ...onSerializationContext_deserialization.cpp | 8 +- typed_python/TypedClosureBuilder.hpp | 4 +- typed_python/_types.cpp | 10 +- typed_python/all.cpp | 3 +- 16 files changed, 1874 insertions(+), 1782 deletions(-) create mode 100644 typed_python/ClosureVariableBinding.cpp create mode 100644 typed_python/ClosureVariableBinding.hpp create mode 100644 typed_python/CompiledSpecialization.hpp create mode 100644 typed_python/FunctionArg.hpp rename typed_python/{FunctionType.cpp => FunctionOverload.cpp} (50%) create mode 100644 typed_python/FunctionOverload.hpp diff --git a/typed_python/ClosureVariableBinding.cpp b/typed_python/ClosureVariableBinding.cpp new file mode 100644 index 000000000..2d1f4dbed --- /dev/null +++ b/typed_python/ClosureVariableBinding.cpp @@ -0,0 +1,87 @@ +#include "PyInstance.hpp" +#include "FunctionType.hpp" + +Instance ClosureVariableBinding::extractValueOrContainingClosure(Type* closureType, instance_ptr data) { + for (long stepIx = 0; stepIx < size(); stepIx++) { + ClosureVariableBindingStep step = (*this)[stepIx]; + + if (step.isFunction()) { + closureType = (Type*)step.getFunction(); + } else + if (step.isNamedField()) { + if (closureType->isNamedTuple()) { + NamedTuple* tupType = (NamedTuple*)closureType; + auto it = tupType->getNameToIndex().find(step.getNamedField()); + if (it == tupType->getNameToIndex().end()) { + throw std::runtime_error( + "Invalid closure: expected NamedTuple to have field " + + step.getNamedField() + " but it doesn't." + ); + } + + closureType = tupType->getTypes()[it->second]; + data = data + tupType->getOffsets()[it->second]; + } else + if (closureType->getTypeCategory() == Type::TypeCategory::catClass) { + Class* clsType = (Class*)closureType; + int index = clsType->getMemberIndex(step.getNamedField().c_str()); + if (index == -1) { + throw std::runtime_error("Can't find a field " + step.getNamedField() + " in class " + clsType->name()); + } + + if (!clsType->checkInitializationFlag(data, index)) { + throw std::runtime_error("Closure field " + step.getNamedField() + " is not populated."); + } + + closureType = clsType->getMemberType(index); + data = clsType->eltPtr(data, index); + } else { + throw std::runtime_error( + "Invalid closure: expected to find a Class or a NamedTuple." + ); + } + } else + if (step.isIndexedField()) { + if (!closureType->isComposite()) { + throw std::runtime_error("Invalid closure: expected a NamedTuple or Tuple but got " + closureType->name()); + } + + CompositeType* tupType = (CompositeType*)closureType; + + if (step.getIndexedField() < 0 || step.getIndexedField() >= tupType->getTypes().size()) { + throw std::runtime_error( + "Invalid closure: index " + format(step.getIndexedField()) + " is out of bounds in closure " + tupType->name() + ); + } + + closureType = tupType->getTypes()[step.getIndexedField()]; + data = data + tupType->getOffsets()[step.getIndexedField()]; + } else + if (step.isCellAccess()) { + if (!(closureType->getTypeCategory() == Type::TypeCategory::catPyCell || + closureType->getTypeCategory() == Type::TypeCategory::catTypedCell)) { + throw std::runtime_error( + "Invalid closure: expected a cell, but got " + + Type::categoryToString(closureType->getTypeCategory()) + ); + } + + if (stepIx + 1 == size()) { + // do nothing, because this function grabs the containing + // closure + } else { + if (closureType->getTypeCategory() == Type::TypeCategory::catPyCell) { + throw std::runtime_error("Corrupt closure encountered: a PyCell should always be the last step"); + } + + // it's a typed closure + data = ((TypedCellType*)closureType)->get(data); + closureType = ((TypedCellType*)closureType)->getHeldType(); + } + } else { + throw std::runtime_error("Corrupt closure variable binding enountered."); + } + } + + return Instance(data, closureType); +} diff --git a/typed_python/ClosureVariableBinding.hpp b/typed_python/ClosureVariableBinding.hpp new file mode 100644 index 000000000..acb2f3651 --- /dev/null +++ b/typed_python/ClosureVariableBinding.hpp @@ -0,0 +1,362 @@ +/****************************************************************************** + Copyright 2017-2022 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Type.hpp" + +class ClosureVariableBindingStep { + enum class BindingType { + FUNCTION = 1, + NAMED_FIELD = 2, + INDEXED_FIELD = 3, + ACCESS_CELL = 4 + }; + + ClosureVariableBindingStep() : + mKind(BindingType::ACCESS_CELL), + mIndexedFieldToAccess(0), + mFunctionToBind(nullptr) + {} + +public: + ClosureVariableBindingStep(Type* bindFunction) : + mKind(BindingType::FUNCTION), + mIndexedFieldToAccess(0), + mFunctionToBind(bindFunction) + {} + + ClosureVariableBindingStep(std::string fieldAccess) : + mKind(BindingType::NAMED_FIELD), + mIndexedFieldToAccess(0), + mFunctionToBind(nullptr), + mNamedFieldToAccess(fieldAccess) + {} + + ClosureVariableBindingStep(int elementAccess) : + mKind(BindingType::INDEXED_FIELD), + mFunctionToBind(nullptr), + mIndexedFieldToAccess(elementAccess) + {} + + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + if (isFunction()) { + v.visitHash(ShaHash(1)); + v.visitTopo(getFunction()); + } else + if (isNamedField()) { + v.visitHash(ShaHash(2) + ShaHash(getNamedField())); + } else + if (isIndexedField()) { + v.visitHash(ShaHash(3) + ShaHash(getIndexedField())); + } else + if (isCellAccess()) { + v.visitHash(ShaHash(4)); + } else { + v.visitErr("Invalid ClosureVariableBindingStep found"); + } + } + + static ClosureVariableBindingStep AccessCell() { + ClosureVariableBindingStep step; + return step; + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (mKind == BindingType::FUNCTION) { + visitor(mFunctionToBind); + } + } + + bool isFunction() const { + return mKind == BindingType::FUNCTION; + } + + bool isNamedField() const { + return mKind == BindingType::NAMED_FIELD; + } + + bool isIndexedField() const { + return mKind == BindingType::INDEXED_FIELD; + } + + bool isCellAccess() const { + return mKind == BindingType::ACCESS_CELL; + } + + Type* getFunction() const { + if (!isFunction()) { + throw std::runtime_error("Binding is not a function"); + } + + return mFunctionToBind; + } + + std::string getNamedField() const { + if (!isNamedField()) { + throw std::runtime_error("Binding is not a named field bindng"); + } + + return mNamedFieldToAccess; + } + + int getIndexedField() const { + if (!isIndexedField()) { + throw std::runtime_error("Binding is not an index field bindng"); + } + + return mIndexedFieldToAccess; + } + + bool operator<(const ClosureVariableBindingStep& step) const { + if (mKind < step.mKind) { + return true; + } + + if (mKind > step.mKind) { + return false; + } + + if (mKind == BindingType::ACCESS_CELL) { + return false; + } + + if (mKind == BindingType::FUNCTION) { + return mFunctionToBind < step.mFunctionToBind; + } + + if (mKind == BindingType::NAMED_FIELD) { + return mNamedFieldToAccess < step.mNamedFieldToAccess; + } + + if (mKind == BindingType::INDEXED_FIELD) { + return mIndexedFieldToAccess < step.mIndexedFieldToAccess; + } + + return false; + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + + if (mKind == BindingType::ACCESS_CELL) { + buffer.writeUnsignedVarintObject(0, 0); + } + else if (mKind == BindingType::FUNCTION) { + buffer.writeUnsignedVarintObject(0, 1); + context.serializeNativeType(mFunctionToBind, buffer, 1); + } + else if (mKind == BindingType::NAMED_FIELD) { + buffer.writeUnsignedVarintObject(0, 2); + buffer.writeStringObject(1, mNamedFieldToAccess); + } + else if (mKind == BindingType::INDEXED_FIELD) { + buffer.writeUnsignedVarintObject(0, 3); + buffer.writeUnsignedVarintObject(1, mIndexedFieldToAccess); + } + + buffer.writeEndCompound(); + } + + template + static ClosureVariableBindingStep deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + ClosureVariableBindingStep out; + + int whichBinding = -1; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + assertWireTypesEqual(wireType, WireType::VARINT); + whichBinding = buffer.readUnsignedVarint(); + + if (whichBinding == 0) { + out = ClosureVariableBindingStep::AccessCell(); + } + } + else if (fieldNumber == 1) { + if (whichBinding == -1) { + throw std::runtime_error("Corrupt ClosureVariableBindingStep"); + } + if (whichBinding == 1) { + out = ClosureVariableBindingStep( + context.deserializeNativeType(buffer, wireType) + ); + } + else if (whichBinding == 2) { + assertWireTypesEqual(wireType, WireType::BYTES); + out = ClosureVariableBindingStep(buffer.readStringObject()); + } + else if (whichBinding == 3) { + assertWireTypesEqual(wireType, WireType::VARINT); + out = ClosureVariableBindingStep(buffer.readUnsignedVarint()); + } + } else { + throw std::runtime_error("Corrupt ClosureVariableBindingStep"); + } + }); + + return out; + } + + +private: + BindingType mKind; + + // this can be a Function or a Forward that will become a function + Type* mFunctionToBind; + + std::string mNamedFieldToAccess; + + int mIndexedFieldToAccess; +}; + + +class ClosureVariableBinding { +public: + ClosureVariableBinding() {} + + ClosureVariableBinding(const std::vector& steps) : + mSteps(new std::vector(steps)) + {} + + ClosureVariableBinding(const std::vector& steps, ClosureVariableBindingStep step) : + mSteps(new std::vector(steps)) + { + mSteps->push_back(step); + } + + ClosureVariableBinding(const ClosureVariableBinding& other) : mSteps(other.mSteps) + {} + + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + v.visitHash(ShaHash(mSteps->size())); + + for (auto step: *mSteps) { + step._visitCompilerVisibleInternals(v); + } + } + + ClosureVariableBinding& operator=(const ClosureVariableBinding& other) { + mSteps = other.mSteps; + return *this; + } + + ClosureVariableBinding operator+(ClosureVariableBindingStep step) { + if (mSteps) { + return ClosureVariableBinding(*mSteps, step); + } + + return ClosureVariableBinding(std::vector(), step); + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + for (long stepIx = 0; stepIx < size(); stepIx++) { + (*this)[stepIx].serialize(context, buffer, stepIx); + } + buffer.writeEndCompound(); + } + + template + static ClosureVariableBinding deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + std::vector steps; + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + steps.push_back(ClosureVariableBindingStep::deserialize(context, buffer, wireType)); + }); + + return ClosureVariableBinding(steps); + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (!mSteps) { + return; + } + + for (auto& step: *mSteps) { + step._visitReferencedTypes(visitor); + } + } + + ClosureVariableBinding withShiftedFrontBinding(long amount) const { + if (!size()) { + throw std::runtime_error("Empty Binding can't be shifted."); + } + + if (!(*this)[0].isIndexedField()) { + throw std::runtime_error("Shifting the first binding only makes sense if it's an indexed lookup"); + } + + std::vector steps; + steps.push_back(ClosureVariableBindingStep((*this)[0].getIndexedField() + amount)); + + for (long k = 1; k < size(); k++) { + steps.push_back((*this)[k]); + } + + return ClosureVariableBinding(steps); + } + + size_t size() const { + if (mSteps) { + return mSteps->size(); + } + + return 0; + } + + bool operator<(const ClosureVariableBinding& other) const { + if (size() < other.size()) { + return true; + } + if (size() > other.size()) { + return false; + } + if (!size()) { + return false; + } + + return *mSteps < *other.mSteps; + } + + ClosureVariableBindingStep operator[](int i) const { + if (i < 0 || i >= size()) { + throw std::runtime_error("ClosureVariableBinding index out of bounds"); + } + + return (*mSteps)[i]; + } + + Instance extractValueOrContainingClosure(Type* closureType, instance_ptr data); + +private: + std::shared_ptr > mSteps; +}; + + +inline ClosureVariableBinding operator+(const ClosureVariableBindingStep& step, const ClosureVariableBinding& binding) { + std::vector steps; + steps.push_back(step); + for (long k = 0; k < binding.size(); k++) { + steps.push_back(binding[k]); + } + return ClosureVariableBinding(steps); +} diff --git a/typed_python/CompiledSpecialization.hpp b/typed_python/CompiledSpecialization.hpp new file mode 100644 index 000000000..560e00a70 --- /dev/null +++ b/typed_python/CompiledSpecialization.hpp @@ -0,0 +1,58 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + + +#include "Type.hpp" +#include + +class CompiledSpecialization { +public: + CompiledSpecialization( + compiled_code_entrypoint funcPtr, + Type* returnType, + const std::vector& argTypes + ) : + mFuncPtr(funcPtr), + mReturnType(returnType), + mArgTypes(argTypes) + {} + + compiled_code_entrypoint getFuncPtr() const { + return mFuncPtr; + } + + Type* getReturnType() const { + return mReturnType; + } + + const std::vector& getArgTypes() const { + return mArgTypes; + } + + bool operator==(const CompiledSpecialization& other) const { + return mFuncPtr == other.mFuncPtr + && mReturnType == other.mReturnType + && mArgTypes == other.mArgTypes + ; + } + +private: + compiled_code_entrypoint mFuncPtr; + Type* mReturnType; + std::vector mArgTypes; +}; diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index 6ab220812..afa894acb 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -872,7 +872,7 @@ class CompilerVisibleObjectVisitor { std::vector > dotAccesses; - Function::Overload::visitCompilerVisibleGlobals( + FunctionOverload::visitCompilerVisibleGlobals( [&](std::string name, PyObject* val) { if (!isSpecialIgnorableName(name)) { visitor.visitNamedTopo(name, val); diff --git a/typed_python/FunctionArg.hpp b/typed_python/FunctionArg.hpp new file mode 100644 index 000000000..1015e5d80 --- /dev/null +++ b/typed_python/FunctionArg.hpp @@ -0,0 +1,174 @@ +/****************************************************************************** + Copyright 2017-2022 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Type.hpp" + +class FunctionArg { +public: + FunctionArg(std::string name, Type* typeFilterOrNull, PyObject* defaultValue, bool isStarArg, bool isKwarg) : + m_name(name), + m_typeFilter(typeFilterOrNull), + m_defaultValue(defaultValue), + m_isStarArg(isStarArg), + m_isKwarg(isKwarg) + { + assert(!(isStarArg && isKwarg)); + } + + std::string getName() const { + return m_name; + } + + PyObject* getDefaultValue() const { + return m_defaultValue; + } + + Type* getTypeFilter() const { + return m_typeFilter; + } + + bool getIsStarArg() const { + return m_isStarArg; + } + + bool getIsKwarg() const { + return m_isKwarg; + } + + bool getIsNormalArg() const { + return !m_isKwarg && !m_isStarArg; + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (m_typeFilter) { + visitor(m_typeFilter); + } + } + + bool operator<(const FunctionArg& other) const { + if (m_name < other.m_name) { + return true; + } + if (m_name > other.m_name) { + return false; + } + if (m_typeFilter < other.m_typeFilter) { + return true; + } + if (m_typeFilter > other.m_typeFilter) { + return false; + } + if (m_defaultValue < other.m_defaultValue) { + return true; + } + if (m_defaultValue > other.m_defaultValue) { + return false; + } + if (m_isStarArg < other.m_isStarArg) { + return true; + } + if (m_isStarArg > other.m_isStarArg) { + return false; + } + if (m_isKwarg < other.m_isKwarg) { + return true; + } + if (m_isKwarg > other.m_isKwarg) { + return false; + } + + return false; + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + + buffer.writeStringObject(0, m_name); + if (m_typeFilter) { + context.serializeNativeType(m_typeFilter, buffer, 1); + } + if (m_defaultValue) { + context.serializePythonObject(m_defaultValue, buffer, 2); + } + buffer.writeUnsignedVarintObject(3, m_isStarArg ? 1 : 0); + buffer.writeUnsignedVarintObject(4, m_isKwarg ? 1 : 0); + + buffer.writeEndCompound(); + } + + template + static FunctionArg deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + std::string name; + Type* typeFilterOrNull = nullptr; + PyObjectHolder defaultValue; + bool isStarArg = false; + bool isKwarg = false; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + assertWireTypesEqual(wireType, WireType::BYTES); + name = buffer.readStringObject(); + } + else if (fieldNumber == 1) { + typeFilterOrNull = context.deserializeNativeType(buffer, wireType); + } + else if (fieldNumber == 2) { + defaultValue.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 3) { + assertWireTypesEqual(wireType, WireType::VARINT); + isStarArg = buffer.readUnsignedVarint(); + } + else if (fieldNumber == 4) { + assertWireTypesEqual(wireType, WireType::VARINT); + isKwarg = buffer.readUnsignedVarint(); + } + }); + + return FunctionArg(name, typeFilterOrNull, defaultValue, isStarArg, isKwarg); + } + + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + v.visitName(m_name); + if (m_defaultValue) { + v.visitTopo((PyObject*)m_defaultValue); + } else { + v.visitHash(ShaHash()); + } + + if (m_typeFilter) { + v.visitTopo(m_typeFilter); + } else { + v.visitHash(ShaHash()); + } + + v.visitHash( + ShaHash((m_isStarArg ? 2 : 1) + (m_isKwarg ? 10: 11)) + ); + } + +private: + std::string m_name; + Type* m_typeFilter; + PyObjectHolder m_defaultValue; + bool m_isStarArg; + bool m_isKwarg; +}; diff --git a/typed_python/FunctionCallArgMapping.hpp b/typed_python/FunctionCallArgMapping.hpp index fd8b9662f..49acbbd07 100644 --- a/typed_python/FunctionCallArgMapping.hpp +++ b/typed_python/FunctionCallArgMapping.hpp @@ -31,23 +31,23 @@ class FunctionCallArgMapping { FunctionCallArgMapping& operator=(const FunctionCallArgMapping&) = delete; public: - class FunctionArg { + class LiveFunctionArg { public: - FunctionArg() : + LiveFunctionArg() : mInstance(), mValid(false), mRawPtr(nullptr) { } - FunctionArg(instance_ptr inData) : + LiveFunctionArg(instance_ptr inData) : mInstance(), mValid(true), mRawPtr(inData) { } - FunctionArg(Instance i) : + LiveFunctionArg(Instance i) : mInstance(i), mValid(true), mRawPtr(nullptr) @@ -75,7 +75,7 @@ class FunctionCallArgMapping { }; - FunctionCallArgMapping(const Function::Overload& overload) : + FunctionCallArgMapping(const FunctionOverload& overload) : mArgs(overload.getArgs()), mCurrentArgumentIx(0), mIsValid(true) @@ -375,7 +375,7 @@ class FunctionCallArgMapping { return res; } - FunctionArg extractArgWithType(int argIx, Type* argType) const { + LiveFunctionArg extractArgWithType(int argIx, Type* argType) const { if (mArgs[argIx].getIsNormalArg()) { try { Type* actualType = PyInstance::extractTypeFrom(mSingleValueArgs[argIx]->ob_type); @@ -384,15 +384,15 @@ class FunctionCallArgMapping { PyInstance* argAsPyInstance = ((PyInstance*)mSingleValueArgs[argIx]); if (argAsPyInstance->mTemporaryRefTo) { - return FunctionArg(argAsPyInstance->mTemporaryRefTo); + return LiveFunctionArg(argAsPyInstance->mTemporaryRefTo); } - return FunctionArg( + return LiveFunctionArg( argAsPyInstance->mContainingInstance ); } - return FunctionArg( + return LiveFunctionArg( Instance::createAndInitialize(argType, [&](instance_ptr p) { PyInstance::copyConstructFromPythonInstance( argType, p, mSingleValueArgs[argIx], ConversionLevel::Signature @@ -402,11 +402,11 @@ class FunctionCallArgMapping { } catch(PythonExceptionSet& s) { // failed to convert, but keep going PyErr_Clear(); - return FunctionArg(); + return LiveFunctionArg(); } catch(...) { // not a valid conversion - return FunctionArg(); + return LiveFunctionArg(); } } else if (mArgs[argIx].getIsStarArg()) { if (argType->getTypeCategory() != Type::TypeCategory::catTuple) { @@ -416,11 +416,11 @@ class FunctionCallArgMapping { Tuple* tup = (Tuple*)argType; if (mStarArgValues.size() != tup->getTypes().size()) { - return FunctionArg(); + return LiveFunctionArg(); } try { - return FunctionArg( + return LiveFunctionArg( Instance::createAndInitialize(tup, [&](instance_ptr p) { tup->constructor(p, [&](instance_ptr subElt, int tupArg) { PyInstance::copyConstructFromPythonInstance( @@ -433,11 +433,11 @@ class FunctionCallArgMapping { } catch(PythonExceptionSet& s) { // failed to convert, but keep going PyErr_Clear(); - return FunctionArg(); + return LiveFunctionArg(); } catch(...) { // not a valid conversion - return FunctionArg(); + return LiveFunctionArg(); } } else if (mArgs[argIx].getIsKwarg()) { if (argType->getTypeCategory() != Type::TypeCategory::catNamedTuple) { @@ -447,17 +447,17 @@ class FunctionCallArgMapping { NamedTuple* tup = (NamedTuple*)argType; if (mKwargValues.size() != tup->getTypes().size()) { - return FunctionArg(); + return LiveFunctionArg(); } for (long k = 0; k < mKwargValues.size(); k++) { if (mKwargValues[k].first != tup->getNames()[k]) { - return FunctionArg(); + return LiveFunctionArg(); } } try { - return FunctionArg( + return LiveFunctionArg( Instance::createAndInitialize(tup, [&](instance_ptr p) { tup->constructor(p, [&](instance_ptr subElt, int tupArg) { PyInstance::copyConstructFromPythonInstance( @@ -472,11 +472,11 @@ class FunctionCallArgMapping { } catch(PythonExceptionSet& s) { // failed to convert, but keep going PyErr_Clear(); - return FunctionArg(); + return LiveFunctionArg(); } catch(...) { // not a valid conversion - return FunctionArg(); + return LiveFunctionArg(); } } @@ -485,7 +485,7 @@ class FunctionCallArgMapping { private: - const std::vector& mArgs; + const std::vector& mArgs; std::vector mSingleValueArgs; std::vector mStarArgValues; diff --git a/typed_python/FunctionType.cpp b/typed_python/FunctionOverload.cpp similarity index 50% rename from typed_python/FunctionType.cpp rename to typed_python/FunctionOverload.cpp index 60a99672f..4151becb6 100644 --- a/typed_python/FunctionType.cpp +++ b/typed_python/FunctionOverload.cpp @@ -1,94 +1,8 @@ -#include "PyInstance.hpp" -#include "FunctionType.hpp" - -Instance ClosureVariableBinding::extractValueOrContainingClosure(Type* closureType, instance_ptr data) { - for (long stepIx = 0; stepIx < size(); stepIx++) { - ClosureVariableBindingStep step = (*this)[stepIx]; - - if (step.isFunction()) { - closureType = (Type*)step.getFunction(); - } else - if (step.isNamedField()) { - if (closureType->isNamedTuple()) { - NamedTuple* tupType = (NamedTuple*)closureType; - auto it = tupType->getNameToIndex().find(step.getNamedField()); - if (it == tupType->getNameToIndex().end()) { - throw std::runtime_error( - "Invalid closure: expected NamedTuple to have field " + - step.getNamedField() + " but it doesn't." - ); - } - - closureType = tupType->getTypes()[it->second]; - data = data + tupType->getOffsets()[it->second]; - } else - if (closureType->getTypeCategory() == Type::TypeCategory::catClass) { - Class* clsType = (Class*)closureType; - int index = clsType->getMemberIndex(step.getNamedField().c_str()); - if (index == -1) { - throw std::runtime_error("Can't find a field " + step.getNamedField() + " in class " + clsType->name()); - } - - if (!clsType->checkInitializationFlag(data, index)) { - throw std::runtime_error("Closure field " + step.getNamedField() + " is not populated."); - } - - closureType = clsType->getMemberType(index); - data = clsType->eltPtr(data, index); - } else { - throw std::runtime_error( - "Invalid closure: expected to find a Class or a NamedTuple." - ); - } - } else - if (step.isIndexedField()) { - if (!closureType->isComposite()) { - throw std::runtime_error("Invalid closure: expected a NamedTuple or Tuple but got " + closureType->name()); - } - - CompositeType* tupType = (CompositeType*)closureType; - - if (step.getIndexedField() < 0 || step.getIndexedField() >= tupType->getTypes().size()) { - throw std::runtime_error( - "Invalid closure: index " + format(step.getIndexedField()) + " is out of bounds in closure " + tupType->name() - ); - } - - closureType = tupType->getTypes()[step.getIndexedField()]; - data = data + tupType->getOffsets()[step.getIndexedField()]; - } else - if (step.isCellAccess()) { - if (!(closureType->getTypeCategory() == Type::TypeCategory::catPyCell || - closureType->getTypeCategory() == Type::TypeCategory::catTypedCell)) { - throw std::runtime_error( - "Invalid closure: expected a cell, but got " - + Type::categoryToString(closureType->getTypeCategory()) - ); - } - - if (stepIx + 1 == size()) { - // do nothing, because this function grabs the containing - // closure - } else { - if (closureType->getTypeCategory() == Type::TypeCategory::catPyCell) { - throw std::runtime_error("Corrupt closure encountered: a PyCell should always be the last step"); - } - - // it's a typed closure - data = ((TypedCellType*)closureType)->get(data); - closureType = ((TypedCellType*)closureType)->getHeldType(); - } - } else { - throw std::runtime_error("Corrupt closure variable binding enountered."); - } - } - - return Instance(data, closureType); -} +#include "FunctionOverload.hpp" /* static */ -PyObject* Function::Overload::buildFunctionObj(Type* closureType, instance_ptr closureData) const { +PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr closureData) const { if (mCachedFunctionObj) { return incref(mCachedFunctionObj); } diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp new file mode 100644 index 000000000..399dc0b7c --- /dev/null +++ b/typed_python/FunctionOverload.hpp @@ -0,0 +1,1128 @@ +/****************************************************************************** + Copyright 2017-2022 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Type.hpp" +#include "TypedCellType.hpp" +#include "ReprAccumulator.hpp" +#include "Format.hpp" +#include "SpecialModuleNames.hpp" +#include "PyInstance.hpp" +#include "ClosureVariableBinding.hpp" +#include "FunctionArg.hpp" +#include "CompiledSpecialization.hpp" + + +class FunctionOverload { +public: + FunctionOverload( + PyObject* pyFuncCode, + PyObject* pyFuncGlobals, + PyObject* pyFuncDefaults, + PyObject* pyFuncAnnotations, + const std::map& pyFuncGlobalsInCells, + const std::vector& pyFuncClosureVarnames, + const std::map closureBindings, + Type* returnType, + PyObject* pySignatureFunction, + const std::vector& args, + Type* methodOf + ) : + mFunctionCode(pyFuncCode), + mFunctionGlobals(pyFuncGlobals), + mFunctionDefaults(pyFuncDefaults), + mFunctionAnnotations(pyFuncAnnotations), + mFunctionGlobalsInCells(pyFuncGlobalsInCells), + mFunctionClosureVarnames(pyFuncClosureVarnames), + mReturnType(returnType), + mSignatureFunction(pySignatureFunction), + mArgs(args), + mHasKwarg(false), + mHasStarArg(false), + mMinPositionalArgs(0), + mMaxPositionalArgs(-1), + mClosureBindings(closureBindings), + mCachedFunctionObj(nullptr), + mMethodOf(methodOf) + { + long argsWithDefaults = 0; + long argsDefinitelyConsuming = 0; + + for (auto arg: mArgs) { + if (arg.getIsStarArg()) { + mHasStarArg = true; + } + else if (arg.getIsKwarg()) { + mHasKwarg = true; + } + else if (arg.getDefaultValue()) { + argsWithDefaults++; + } else { + argsDefinitelyConsuming++; + } + } + + mMinPositionalArgs = argsDefinitelyConsuming; + if (!mHasStarArg) { + mMaxPositionalArgs = argsDefinitelyConsuming + argsWithDefaults; + } + + increfAllPyObjects(); + } + + FunctionOverload(const FunctionOverload& other) { + other.increfAllPyObjects(); + + mFunctionCode = other.mFunctionCode; + mFunctionGlobals = other.mFunctionGlobals; + mFunctionDefaults = other.mFunctionDefaults; + mFunctionAnnotations = other.mFunctionAnnotations; + mSignatureFunction = other.mSignatureFunction; + mMethodOf = other.mMethodOf; + + mFunctionClosureVarnames = other.mFunctionClosureVarnames; + + mClosureBindings = other.mClosureBindings; + mReturnType = other.mReturnType; + mArgs = other.mArgs; + mCompiledSpecializations = other.mCompiledSpecializations; + + mHasStarArg = other.mHasStarArg; + mHasKwarg = other.mHasKwarg; + mMinPositionalArgs = other.mMinPositionalArgs; + mMaxPositionalArgs = other.mMaxPositionalArgs; + + mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; + + mCachedFunctionObj = other.mCachedFunctionObj; + } + + ~FunctionOverload() { + decrefAllPyObjects(); + } + + FunctionOverload withShiftedFrontClosureBindings(long shiftAmount) const { + std::map bindings; + for (auto nameAndBinding: mClosureBindings) { + bindings[nameAndBinding.first] = nameAndBinding.second.withShiftedFrontBinding(shiftAmount); + } + + return FunctionOverload( + mFunctionCode, + mFunctionGlobals, + mFunctionDefaults, + mFunctionAnnotations, + mFunctionGlobalsInCells, + mFunctionClosureVarnames, + bindings, + mReturnType, + mSignatureFunction, + mArgs, + mMethodOf + ); + } + + FunctionOverload withMethodOf(Type* methodOf) const { + return FunctionOverload( + mFunctionCode, + mFunctionGlobals, + mFunctionDefaults, + mFunctionAnnotations, + mFunctionGlobalsInCells, + mFunctionClosureVarnames, + mClosureBindings, + mReturnType, + mSignatureFunction, + mArgs, + methodOf + ); + } + + FunctionOverload withClosureBindings(const std::map &bindings) const { + return FunctionOverload( + mFunctionCode, + mFunctionGlobals, + mFunctionDefaults, + mFunctionAnnotations, + mFunctionGlobalsInCells, + mFunctionClosureVarnames, + bindings, + mReturnType, + mSignatureFunction, + mArgs, + mMethodOf + ); + } + + std::string toString() const { + std::ostringstream str; + + str << "("; + + for (long k = 0; k < mArgs.size(); k++) { + if (k) { + str << ", "; + } + + if (mArgs[k].getIsStarArg()) { + str << "*"; + } + + if (mArgs[k].getIsKwarg()) { + str << "**"; + } + + str << mArgs[k].getName(); + + if (mArgs[k].getDefaultValue()) { + str << "=..."; + } + + if (mArgs[k].getTypeFilter()) { + str << ": " << mArgs[k].getTypeFilter()->name(); + } + } + + str << ")"; + + if (mReturnType) { + str << " -> " << mReturnType->name(); + } + + return str.str(); + } + + // return the FunctionArg* that a positional argument would map to, or 'nullptr' if + // it wouldn't + const FunctionArg* argForPositionalArgument(long argIx) const { + if (argIx >= mArgs.size()) { + return nullptr; + } + + if (mArgs[argIx].getIsStarArg() || mArgs[argIx].getIsKwarg()) { + return nullptr; + } + + return &mArgs[argIx]; + } + + // can we possibly match 'argCount' positional arguments? + bool couldMatchPositionalCount(long argCount) const { + return argCount >= mMinPositionalArgs && argCount < mMaxPositionalArgs; + } + + Type* getReturnType() const { + return mReturnType; + } + + const std::vector& getFunctionClosureVarnames() const { + return mFunctionClosureVarnames; + } + + const std::vector& getArgs() const { + return mArgs; + } + + void finalizeTypeConcrete() { + // ensure that the Function object we represent is in sync with + // our actual return and argument types. It would be better to not have + // any direct references to the python interpreter in this class and + // rebuild the concrete function object on demand later... + if (mReturnType) { + if (mFunctionAnnotations + && PyDict_Check(mFunctionAnnotations) + ) { + PyDict_SetItemString( + mFunctionAnnotations, + "return", + (PyObject*)PyInstance::typeObj(mReturnType) + ); + } + } + for (auto& a: mArgs) { + Type* typeFilter = a.getTypeFilter(); + + if (typeFilter + && mFunctionAnnotations + && PyDict_Check(mFunctionAnnotations) + ) { + PyDict_SetItemString( + mFunctionAnnotations, + a.getName().c_str(), + (PyObject*)PyInstance::typeObj(a.getTypeFilter()) + ); + } + } + } + + template + void _visitReferencedTypes(const visitor_type& visitor) { + // we need to keep mArgs and mReturnType in sync with + // mAnnotations, so if we change one of our types we + // need to update the resulting dictionary as well. + if (mReturnType) { + visitor(mReturnType); + } + for (auto& a: mArgs) { + a._visitReferencedTypes(visitor); + } + + for (auto& varnameAndBinding: mClosureBindings) { + varnameAndBinding.second._visitReferencedTypes(visitor); + } + + if (mMethodOf) { + visitor(mMethodOf); + } + + for (auto nameAndGlobal: mFunctionGlobalsInCells) { + if (!PyCell_Check(nameAndGlobal.second)) { + throw std::runtime_error( + "A global in mFunctionGlobalsInCells is somehow not a cell" + ); + } + + PyObject* cellContents = PyCell_Get(nameAndGlobal.second); + + if (cellContents && PyType_Check(cellContents)) { + Type* t = PyInstance::extractTypeFrom(cellContents); + if (t) { + visitor(t); + } + } + } + + _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { + if (PyType_Check(val)) { + Type* t = PyInstance::extractTypeFrom(val); + if (t) { + visitor(t); + } + } + }); + } + + template + void _visitCompilerVisibleInternals(const visitor_type& visitor) { + visitor.visitTopo(mFunctionCode); + + if (mFunctionAnnotations) { + visitor.visitHash(ShaHash(2)); + + if (PyDict_CheckExact(mFunctionAnnotations)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(mFunctionAnnotations, &pos, &key, &value)) { + visitor.visitTopo(key); + visitor.visitTopo(value); + } + } + + visitor.visitHash(ShaHash(2)); + } + + if (mSignatureFunction) { + visitor.visitHash(ShaHash(1)); + visitor.visitTopo(mSignatureFunction); + } else { + visitor.visitHash(ShaHash(0)); + } + + if (mFunctionDefaults) { + visitor.visitHash(ShaHash(2)); + + if (PyDict_CheckExact(mFunctionDefaults)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(mFunctionDefaults, &pos, &key, &value)) { + visitor.visitTopo(key); + visitor.visitTopo(value); + } + } + + visitor.visitHash(ShaHash(2)); + } + + visitor.visitHash(ShaHash(mFunctionGlobalsInCells.size())); + + for (auto nameAndGlobal: mFunctionGlobalsInCells) { + if (!PyCell_Check(nameAndGlobal.second)) { + throw std::runtime_error( + "A global in mFunctionGlobalsInCells is somehow not a cell" + ); + } + + visitor.visitNamedTopo( + nameAndGlobal.first, + nameAndGlobal.second + ); + } + + if (mReturnType) { + visitor.visitTopo(mReturnType); + } else { + visitor.visitHash(ShaHash()); + } + + if (mMethodOf) { + visitor.visitTopo(mMethodOf); + } else { + visitor.visitHash(ShaHash()); + } + + visitor.visitHash(ShaHash(mClosureBindings.size())); + for (auto nameAndClosure: mClosureBindings) { + visitor.visitName(nameAndClosure.first); + nameAndClosure.second._visitCompilerVisibleInternals(visitor); + } + + visitor.visitHash(ShaHash(mArgs.size())); + + for (auto a: mArgs) { + a._visitCompilerVisibleInternals(visitor); + } + + visitor.visitHash(ShaHash(1)); + + _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { + visitor.visitNamedTopo(name, val); + }); + + visitor.visitHash(ShaHash(1)); + } + + template + static void visitCompilerVisibleGlobals( + const visitor_type& visitor, + PyCodeObject* code, + PyObject* globals + ) { + std::vector > dotAccesses; + + extractDottedGlobalAccessesFromCode(code, dotAccesses); + + auto visitSequence = [&](const std::vector& sequence) { + PyObjectHolder curObj; + std::string curName; + + for (PyObject* name: sequence) { + std::string nameStr = PyUnicode_AsUTF8(name); + + if (isSpecialIgnorableName(nameStr)) { + break; + } + + if (!curObj) { + curName = nameStr; + } else { + curName = curName + "." + nameStr; + } + + if (!curObj) { + // this is a lookup in the global dict + curObj.set(PyDict_GetItem(globals, name)); + + if (!curObj) { + // this is an invalid global lookup, which is OK. no need to hash anything. + PyErr_Clear(); + return; + } + } else { + // we're looking up an attribute of this object. We only want to look into modules. + if (PyModule_CheckExact(curObj) && PyObject_HasAttr(curObj, name)) { + PyObjectStealer moduleMember(PyObject_GetAttr(curObj, name)); + + curObj.steal(PyObject_GetAttr(curObj, name)); + if (!curObj) { + // this is an invalid module member lookup. We can just bail. + PyErr_Clear(); + return; + } + } else { + break; + } + } + } + + // also visit at the end of the sequence + if (curObj) { + visitor(curName, (PyObject*)curObj); + } + }; + + for (auto& sequence: dotAccesses) { + visitSequence(sequence); + } + } + + template + void _visitCompilerVisibleGlobals(const visitor_type& visitor) { + visitCompilerVisibleGlobals(visitor, (PyCodeObject*)mFunctionCode, mFunctionGlobals); + } + + template + void _visitContainedTypes(const visitor_type& visitor) { + } + + const std::vector& getCompiledSpecializations() const { + return mCompiledSpecializations; + } + + void addCompiledSpecialization(compiled_code_entrypoint e, Type* returnType, const std::vector& argTypes) { + CompiledSpecialization newSpec = CompiledSpecialization(e,returnType,argTypes); + + for (auto& spec: mCompiledSpecializations) { + if (spec == newSpec) { + return; + } + } + + mCompiledSpecializations.push_back(newSpec); + } + + void touchCompiledSpecializations() { + //force the memory for the compiled specializations to move. + std::vector other = mCompiledSpecializations; + std::swap(mCompiledSpecializations, other); + } + + bool operator<(const FunctionOverload& other) const { + if (mFunctionCode < other.mFunctionCode) { return true; } + if (mFunctionCode > other.mFunctionCode) { return false; } + + if (mFunctionGlobals < other.mFunctionGlobals) { return true; } + if (mFunctionGlobals > other.mFunctionGlobals) { return false; } + + if (mFunctionGlobalsInCells < other.mFunctionGlobalsInCells) { return true; } + if (mFunctionGlobalsInCells > other.mFunctionGlobalsInCells) { return false; } + + if (mClosureBindings < other.mClosureBindings) { return true; } + if (mClosureBindings > other.mClosureBindings) { return false; } + + if (mReturnType < other.mReturnType) { return true; } + if (mReturnType > other.mReturnType) { return false; } + + if (mArgs < other.mArgs) { return true; } + if (mArgs > other.mArgs) { return false; } + + if (mMethodOf > other.mMethodOf) { return true; } + if (mMethodOf < other.mMethodOf) { return false; } + + return false; + } + + const std::map& getClosureVariableBindings() const { + return mClosureBindings; + } + + PyObject* getFunctionCode() const { + return mFunctionCode; + } + + PyObject* getFunctionDefaults() const { + return mFunctionDefaults; + } + + PyObject* getFunctionAnnotations() const { + return mFunctionAnnotations; + } + + Type* getMethodOf() const { + return mMethodOf; + } + + PyObject* getSignatureFunction() const { + return mSignatureFunction; + } + + PyObject* getFunctionGlobals() const { + return mFunctionGlobals; + } + + const std::map& getFunctionGlobalsInCells() const { + return mFunctionGlobalsInCells; + } + + /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. + + In cases where we write something like 'x.y.z' the compiler shouldn't have a reference to 'x', + just to whatever 'x.y.z' refers to. + + This transformation just figures out what the dotting sequences are. + */ + static void extractDottedGlobalAccessesFromCode(PyCodeObject* code, std::vector >& outSequences) { + uint8_t* bytes; + Py_ssize_t bytecount; + + static PyObject* moduleHashName = PyUnicode_FromString("__module_hash__"); + outSequences.push_back(std::vector({moduleHashName})); + + PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); + + long opcodeCount = bytecount / 2; + + // opcodes are encoded in the low byte + auto opcodeFor = [&](int i) { return bytes[i * 2]; }; + + // opcode targets are encoded in the high byte + auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; + + const uint8_t LOAD_ATTR = 106; + const uint8_t LOAD_GLOBAL = 116; + const uint8_t DELETE_GLOBAL = 98; + const uint8_t STORE_GLOBAL = 97; + const uint8_t LOAD_METHOD = 160; + + + std::vector curDotSequence; + for (long ix = 0; ix < opcodeCount; ix++) { + // if we're loading an attr on an existing sequence, just make it bigger + if ((opcodeFor(ix) == LOAD_ATTR || opcodeFor(ix) == LOAD_METHOD) && curDotSequence.size()) { + curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); + } else if (curDotSequence.size()) { + // any other operation should flush the buffer + outSequences.push_back(curDotSequence); + curDotSequence.clear(); + } + + // if we're loading a global, we start a new sequence + if (opcodeFor(ix) == LOAD_GLOBAL) { + curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); + } else if ( + opcodeFor(ix) == STORE_GLOBAL + || opcodeFor(ix) == DELETE_GLOBAL + ) { + outSequences.push_back({PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))}); + } + } + + // flush the buffer if we have something + if (curDotSequence.size()) { + outSequences.push_back(curDotSequence); + } + + // recurse into sub code objects + iterate(code->co_consts, [&](PyObject* o) { + if (PyCode_Check(o)) { + extractDottedGlobalAccessesFromCode((PyCodeObject*)o, outSequences); + } + }); + } + + static void extractGlobalAccessesFromCode(PyCodeObject* code, std::set& outAccesses) { + uint8_t* bytes; + Py_ssize_t bytecount; + + PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); + + long opcodeCount = bytecount / 2; + + // opcodes are encoded in the low byte + auto opcodeFor = [&](int i) { return bytes[i * 2]; }; + + // opcode targets are encoded in the high byte + auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; + + const uint8_t LOAD_GLOBAL = 116; + const uint8_t DELETE_GLOBAL = 98; + const uint8_t STORE_GLOBAL = 97; + + for (long ix = 0; ix < opcodeCount; ix++) { + // if we're loading a global, we start a new sequence + if (opcodeFor(ix) == LOAD_GLOBAL) { + PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); + if (!PyUnicode_Check(name)) { + throw std::runtime_error("Function had a non-string object in co_names"); + } + outAccesses.insert(PyUnicode_AsUTF8(name)); + } else if ( + opcodeFor(ix) == STORE_GLOBAL + || opcodeFor(ix) == DELETE_GLOBAL + ) { + PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); + if (!PyUnicode_Check(name)) { + throw std::runtime_error("Function had a non-string object in co_names"); + } + outAccesses.insert(PyUnicode_AsUTF8(name)); + } + } + + // recurse into sub code objects + iterate(code->co_consts, [&](PyObject* o) { + if (PyCode_Check(o)) { + extractGlobalAccessesFromCode((PyCodeObject*)o, outAccesses); + } + }); + + outAccesses.insert("__module_hash__"); + } + + void extractGlobalAccessesFromCodeIncludingCells( + std::set& outNames + ) const { + for (auto nameAndGlobal: mFunctionGlobalsInCells) { + outNames.insert(nameAndGlobal.first); + } + + outNames.insert("__module_hash__"); + + extractGlobalAccessesFromCode((PyCodeObject*)mFunctionCode, outNames); + } + + void setGlobals(PyObject* globals) { + decref(mFunctionGlobals); + mFunctionGlobals = incref(globals); + } + + bool symbolIsUnresolved(std::string name, bool insistResolved) { + if (mClosureBindings.find(name) != mClosureBindings.end()) { + return false; + } + + auto it = mFunctionGlobalsInCells.find(name); + if (it != mFunctionGlobalsInCells.end()) { + if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + if (insistResolved) { + PyObject* o = PyCell_Get(it->second); + Type* t = PyInstance::extractTypeFrom(o); + if (t && t->isForwardDefined()) { + return true; + } + } + + return false; + } + + return true; + } + + if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { + if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { + if (insistResolved) { + PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + Type* t = PyInstance::extractTypeFrom(o); + if (t && t->isForwardDefined()) { + return true; + } + } + + return false; + } + + PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); + if (builtins && PyDict_GetItemString(builtins, name.c_str())) { + return false; + } + } + + return true; + } + + void autoresolveGlobal( + std::string name, + const std::set& resolvedForwards + ) { + auto it = mFunctionGlobalsInCells.find(name); + if (it != mFunctionGlobalsInCells.end()) { + if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + if (PyType_Check(PyCell_GET(it->second))) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)PyCell_Get(it->second) + ); + + if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + PyCell_Set( + it->second, + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + } + } + + return; + } + + PyObject* dictVal = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + if (dictVal && PyType_Check(dictVal)) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)dictVal + ); + + if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + PyDict_SetItemString( + mFunctionGlobals, + name.c_str(), + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + } + } + + void autoresolveGlobals(const std::set& resolvedForwards) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__") { + autoresolveGlobal(nameStr, resolvedForwards); + } + } + } + + bool hasUnresolvedSymbols(bool insistResolved) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { + return true; + } + } + + return false; + } + + std::string firstUnresolvedSymbol(bool insistResolved) { + std::set allNames; + extractGlobalAccessesFromCodeIncludingCells(allNames); + + for (auto nameStr: allNames) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { + return nameStr; + } + } + + return ""; + } + + PyObject* getUsedGlobals() const { + // restrict the globals to contain only the values it references. + PyObject* result = PyDict_New(); + + std::set allNamesString; + extractGlobalAccessesFromCodeIncludingCells(allNamesString); + + for (auto nameStr: allNamesString) { + if (nameStr != "__module_hash__" && mClosureBindings.find(nameStr) != mClosureBindings.end()) { + throw std::runtime_error( + "Somehow we have both a closure binding and a global for " + + nameStr + ); + } + } + + // iterate mFunctionGlobals, keeping any where the name is in allNames + // note we split on '.' and take the first part so that if a module + // like lxml.etree is included, and we use 'lxml', we'll take the reference + // to etree as well. This ensures that submodules in anonymously serialized + // code can get pulled along. + PyObject *key, *value; + Py_ssize_t pos = 0; + + // place them in sorted order + std::map > toCopy; + + while (PyDict_Next(mFunctionGlobals, &pos, &key, &value)) { + if (PyUnicode_Check(key)) { + std::string globalName = PyUnicode_AsUTF8(key); + std::string shortGlobalName = globalName; + + size_t indexOfDot = shortGlobalName.find('.'); + if (indexOfDot != std::string::npos) { + shortGlobalName = shortGlobalName.substr(0, indexOfDot); + } + + if (allNamesString.find(shortGlobalName) != allNamesString.end()) { + toCopy[globalName] = std::make_pair(key, value); + } + } + } + + for (auto& nameandKV: toCopy) { + if (PyDict_SetItem(result, nameandKV.second.first, nameandKV.second.second)) { + throw PythonExceptionSet(); + } + } + + PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); + if (builtins) { + PyDict_SetItemString(result, "__builtins__", builtins); + } + + return result; + } + + // create a new function object for this closure (or cache it + // if we have no closure) + PyObject* buildFunctionObj(Type* closureType, instance_ptr closureData) const; + + FunctionOverload& operator=(const FunctionOverload& other) { + other.increfAllPyObjects(); + decrefAllPyObjects(); + + mFunctionCode = other.mFunctionCode; + mFunctionGlobals = other.mFunctionGlobals; + mFunctionDefaults = other.mFunctionDefaults; + mFunctionAnnotations = other.mFunctionAnnotations; + mSignatureFunction = other.mSignatureFunction; + + mMethodOf = other.mMethodOf; + + mFunctionClosureVarnames = other.mFunctionClosureVarnames; + + mClosureBindings = other.mClosureBindings; + mReturnType = other.mReturnType; + mArgs = other.mArgs; + mCompiledSpecializations = other.mCompiledSpecializations; + + mHasStarArg = other.mHasStarArg; + mHasKwarg = other.mHasKwarg; + mMinPositionalArgs = other.mMinPositionalArgs; + mMaxPositionalArgs = other.mMaxPositionalArgs; + mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; + + mCachedFunctionObj = other.mCachedFunctionObj; + + return *this; + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + buffer.writeBeginCompound(fieldNumber); + + context.serializePythonObject(mFunctionCode, buffer, 0); + + if (mFunctionDefaults) { + context.serializePythonObject(mFunctionDefaults, buffer, 2); + } + + if (mFunctionAnnotations) { + context.serializePythonObject(mFunctionAnnotations, buffer, 3); + } + + buffer.writeBeginCompound(4); + int stringIx = 0; + for (auto varname: mFunctionClosureVarnames) { + buffer.writeStringObject(stringIx++, varname); + } + buffer.writeEndCompound(); + + buffer.writeBeginCompound(5); + int varIx = 0; + for (auto nameAndCell: mFunctionGlobalsInCells) { + buffer.writeStringObject(varIx++, nameAndCell.first); + context.serializePythonObject(nameAndCell.second, buffer, varIx++); + } + buffer.writeEndCompound(); + + buffer.writeBeginCompound(6); + int closureBindingIx = 0; + for (auto nameAndBinding: mClosureBindings) { + buffer.writeStringObject(closureBindingIx++, nameAndBinding.first); + nameAndBinding.second.serialize(context, buffer, closureBindingIx++); + } + buffer.writeEndCompound(); + + if (mReturnType) { + context.serializeNativeType(mReturnType, buffer, 7); + } + + buffer.writeBeginCompound(8); + + int argIx = 0; + for (auto& arg: mArgs) { + arg.serialize(context, buffer, argIx++); + } + + buffer.writeEndCompound(); + + if (mSignatureFunction) { + context.serializePythonObject(mSignatureFunction, buffer, 9); + } + + if (mMethodOf) { + context.serializeNativeType(mMethodOf, buffer, 10); + } + + buffer.writeEndCompound(); + } + + template + static FunctionOverload deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + PyObjectHolder functionCode; + PyObjectHolder functionGlobals; + PyObjectHolder functionAnnotations; + PyObjectHolder functionDefaults; + PyObjectHolder functionSignature; + std::vector closureVarnames; + std::map functionGlobalsInCells; + std::map functionGlobalsInCellsRaw; + std::map closureBindings; + Type* returnType = nullptr; + Type* methodOf = nullptr; + std::vector args; + + functionGlobals.steal(PyDict_New()); + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + functionCode.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 2) { + functionDefaults.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 3) { + functionAnnotations.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 4) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + assertWireTypesEqual(wireType, WireType::BYTES); + closureVarnames.push_back(buffer.readStringObject()); + }); + } + else if (fieldNumber == 5) { + std::string last; + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber % 2 == 0) { + assertWireTypesEqual(wireType, WireType::BYTES); + last = buffer.readStringObject(); + } else { + if (last == "") { + throw std::runtime_error("Corrupt Function closure encountered"); + } + functionGlobalsInCells[last].steal(context.deserializePythonObject(buffer, wireType)); + functionGlobalsInCellsRaw[last] = functionGlobalsInCells[last]; + last = ""; + } + }); + } + else if (fieldNumber == 6) { + std::string last; + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber % 2 == 0) { + assertWireTypesEqual(wireType, WireType::BYTES); + last = buffer.readStringObject(); + } else { + closureBindings[last] = ClosureVariableBinding::deserialize(context, buffer, wireType); + } + }); + } + else if (fieldNumber == 7) { + returnType = context.deserializeNativeType(buffer, wireType); + } + else if (fieldNumber == 8) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + args.push_back(FunctionArg::deserialize(context, buffer, wireType)); + }); + } + else if (fieldNumber == 9) { + functionSignature.steal(context.deserializePythonObject(buffer, wireType)); + } + else if (fieldNumber == 10) { + methodOf = context.deserializeNativeType(buffer, wireType); + } + }); + + return FunctionOverload( + functionCode, + functionGlobals, + functionDefaults, + functionAnnotations, + functionGlobalsInCellsRaw, + closureVarnames, + closureBindings, + returnType, + functionSignature, + args, + methodOf + ); + } + + void increfAllPyObjects() const { + incref(mFunctionCode); + incref(mFunctionGlobals); + incref(mFunctionDefaults); + incref(mFunctionAnnotations); + incref(mSignatureFunction); + + for (auto nameAndOther: mFunctionGlobalsInCells) { + incref(nameAndOther.second); + } + } + + void decrefAllPyObjects() { + decref(mFunctionCode); + decref(mFunctionGlobals); + decref(mFunctionDefaults); + decref(mFunctionAnnotations); + decref(mSignatureFunction); + + for (auto nameAndOther: mFunctionGlobalsInCells) { + decref(nameAndOther.second); + } + } + +private: + PyObject* mFunctionCode; + + PyObject* mFunctionGlobals; + + // globals that are stored in cells. This happens when class objects + // are defined inside of function scopes. We assume that anything in their + // closure is global (and therefore constant) but it may not be defined yet, + // so we can't just pull the value out and stick it in the function closure + // itself. Each value in the map is guaranteed to be a 'cell' object. + std::map mFunctionGlobalsInCells; + + // the order (by name) of the variables in the __closure__ of the original + // function. This is the order that the python code will expect. + std::vector mFunctionClosureVarnames; + + PyObject* mFunctionDefaults; + + PyObject* mFunctionAnnotations; + + PyObject* mSignatureFunction; + + // note that we are deliberately leaking this value because Overloads get + // stashed in static std::map memos anyways. + mutable PyObject* mCachedFunctionObj; + + std::map mClosureBindings; + + Type* mReturnType; + + // if we are a method of a class, what class? Used to + Type* mMethodOf; + + std::vector mArgs; + + // in compiled code, the closure arguments get passed in front of the + // actual function arguments + std::vector mCompiledSpecializations; + + bool mHasStarArg; + bool mHasKwarg; + size_t mMinPositionalArgs; + size_t mMaxPositionalArgs; +}; diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 5431e99c9..610f7f366 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -22,352 +22,12 @@ #include "Format.hpp" #include "SpecialModuleNames.hpp" #include "PyInstance.hpp" +#include "ClosureVariableBinding.hpp" +#include "FunctionArg.hpp" +#include "FunctionOverload.hpp" class Function; -class ClosureVariableBindingStep { - enum class BindingType { - FUNCTION = 1, - NAMED_FIELD = 2, - INDEXED_FIELD = 3, - ACCESS_CELL = 4 - }; - - ClosureVariableBindingStep() : - mKind(BindingType::ACCESS_CELL), - mIndexedFieldToAccess(0), - mFunctionToBind(nullptr) - {} - -public: - ClosureVariableBindingStep(Type* bindFunction) : - mKind(BindingType::FUNCTION), - mIndexedFieldToAccess(0), - mFunctionToBind(bindFunction) - {} - - ClosureVariableBindingStep(std::string fieldAccess) : - mKind(BindingType::NAMED_FIELD), - mIndexedFieldToAccess(0), - mFunctionToBind(nullptr), - mNamedFieldToAccess(fieldAccess) - {} - - ClosureVariableBindingStep(int elementAccess) : - mKind(BindingType::INDEXED_FIELD), - mFunctionToBind(nullptr), - mIndexedFieldToAccess(elementAccess) - {} - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - if (isFunction()) { - v.visitHash(ShaHash(1)); - v.visitTopo(getFunction()); - } else - if (isNamedField()) { - v.visitHash(ShaHash(2) + ShaHash(getNamedField())); - } else - if (isIndexedField()) { - v.visitHash(ShaHash(3) + ShaHash(getIndexedField())); - } else - if (isCellAccess()) { - v.visitHash(ShaHash(4)); - } else { - v.visitErr("Invalid ClosureVariableBindingStep found"); - } - } - - static ClosureVariableBindingStep AccessCell() { - ClosureVariableBindingStep step; - return step; - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - if (mKind == BindingType::FUNCTION) { - visitor(mFunctionToBind); - } - } - - bool isFunction() const { - return mKind == BindingType::FUNCTION; - } - - bool isNamedField() const { - return mKind == BindingType::NAMED_FIELD; - } - - bool isIndexedField() const { - return mKind == BindingType::INDEXED_FIELD; - } - - bool isCellAccess() const { - return mKind == BindingType::ACCESS_CELL; - } - - Type* getFunction() const { - if (!isFunction()) { - throw std::runtime_error("Binding is not a function"); - } - - return mFunctionToBind; - } - - std::string getNamedField() const { - if (!isNamedField()) { - throw std::runtime_error("Binding is not a named field bindng"); - } - - return mNamedFieldToAccess; - } - - int getIndexedField() const { - if (!isIndexedField()) { - throw std::runtime_error("Binding is not an index field bindng"); - } - - return mIndexedFieldToAccess; - } - - bool operator<(const ClosureVariableBindingStep& step) const { - if (mKind < step.mKind) { - return true; - } - - if (mKind > step.mKind) { - return false; - } - - if (mKind == BindingType::ACCESS_CELL) { - return false; - } - - if (mKind == BindingType::FUNCTION) { - return mFunctionToBind < step.mFunctionToBind; - } - - if (mKind == BindingType::NAMED_FIELD) { - return mNamedFieldToAccess < step.mNamedFieldToAccess; - } - - if (mKind == BindingType::INDEXED_FIELD) { - return mIndexedFieldToAccess < step.mIndexedFieldToAccess; - } - - return false; - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - - if (mKind == BindingType::ACCESS_CELL) { - buffer.writeUnsignedVarintObject(0, 0); - } - else if (mKind == BindingType::FUNCTION) { - buffer.writeUnsignedVarintObject(0, 1); - context.serializeNativeType(mFunctionToBind, buffer, 1); - } - else if (mKind == BindingType::NAMED_FIELD) { - buffer.writeUnsignedVarintObject(0, 2); - buffer.writeStringObject(1, mNamedFieldToAccess); - } - else if (mKind == BindingType::INDEXED_FIELD) { - buffer.writeUnsignedVarintObject(0, 3); - buffer.writeUnsignedVarintObject(1, mIndexedFieldToAccess); - } - - buffer.writeEndCompound(); - } - - template - static ClosureVariableBindingStep deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - ClosureVariableBindingStep out; - - int whichBinding = -1; - - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber == 0) { - assertWireTypesEqual(wireType, WireType::VARINT); - whichBinding = buffer.readUnsignedVarint(); - - if (whichBinding == 0) { - out = ClosureVariableBindingStep::AccessCell(); - } - } - else if (fieldNumber == 1) { - if (whichBinding == -1) { - throw std::runtime_error("Corrupt ClosureVariableBindingStep"); - } - if (whichBinding == 1) { - out = ClosureVariableBindingStep( - context.deserializeNativeType(buffer, wireType) - ); - } - else if (whichBinding == 2) { - assertWireTypesEqual(wireType, WireType::BYTES); - out = ClosureVariableBindingStep(buffer.readStringObject()); - } - else if (whichBinding == 3) { - assertWireTypesEqual(wireType, WireType::VARINT); - out = ClosureVariableBindingStep(buffer.readUnsignedVarint()); - } - } else { - throw std::runtime_error("Corrupt ClosureVariableBindingStep"); - } - }); - - return out; - } - - -private: - BindingType mKind; - - // this can be a Function or a Forward that will become a function - Type* mFunctionToBind; - - std::string mNamedFieldToAccess; - - int mIndexedFieldToAccess; -}; - - -class ClosureVariableBinding { -public: - ClosureVariableBinding() {} - - ClosureVariableBinding(const std::vector& steps) : - mSteps(new std::vector(steps)) - {} - - ClosureVariableBinding(const std::vector& steps, ClosureVariableBindingStep step) : - mSteps(new std::vector(steps)) - { - mSteps->push_back(step); - } - - ClosureVariableBinding(const ClosureVariableBinding& other) : mSteps(other.mSteps) - {} - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitHash(ShaHash(mSteps->size())); - - for (auto step: *mSteps) { - step._visitCompilerVisibleInternals(v); - } - } - - ClosureVariableBinding& operator=(const ClosureVariableBinding& other) { - mSteps = other.mSteps; - return *this; - } - - ClosureVariableBinding operator+(ClosureVariableBindingStep step) { - if (mSteps) { - return ClosureVariableBinding(*mSteps, step); - } - - return ClosureVariableBinding(std::vector(), step); - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - for (long stepIx = 0; stepIx < size(); stepIx++) { - (*this)[stepIx].serialize(context, buffer, stepIx); - } - buffer.writeEndCompound(); - } - - template - static ClosureVariableBinding deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - std::vector steps; - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - steps.push_back(ClosureVariableBindingStep::deserialize(context, buffer, wireType)); - }); - - return ClosureVariableBinding(steps); - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - if (!mSteps) { - return; - } - - for (auto& step: *mSteps) { - step._visitReferencedTypes(visitor); - } - } - - ClosureVariableBinding withShiftedFrontBinding(long amount) const { - if (!size()) { - throw std::runtime_error("Empty Binding can't be shifted."); - } - - if (!(*this)[0].isIndexedField()) { - throw std::runtime_error("Shifting the first binding only makes sense if it's an indexed lookup"); - } - - std::vector steps; - steps.push_back(ClosureVariableBindingStep((*this)[0].getIndexedField() + amount)); - - for (long k = 1; k < size(); k++) { - steps.push_back((*this)[k]); - } - - return ClosureVariableBinding(steps); - } - - size_t size() const { - if (mSteps) { - return mSteps->size(); - } - - return 0; - } - - bool operator<(const ClosureVariableBinding& other) const { - if (size() < other.size()) { - return true; - } - if (size() > other.size()) { - return false; - } - if (!size()) { - return false; - } - - return *mSteps < *other.mSteps; - } - - ClosureVariableBindingStep operator[](int i) const { - if (i < 0 || i >= size()) { - throw std::runtime_error("ClosureVariableBinding index out of bounds"); - } - - return (*mSteps)[i]; - } - - Instance extractValueOrContainingClosure(Type* closureType, instance_ptr data); - -private: - std::shared_ptr > mSteps; -}; - - -inline ClosureVariableBinding operator+(const ClosureVariableBindingStep& step, const ClosureVariableBinding& binding) { - std::vector steps; - steps.push_back(step); - for (long k = 0; k < binding.size(); k++) { - steps.push_back(binding[k]); - } - return ClosureVariableBinding(steps); -} - PyDoc_STRVAR(Function_doc, "Function(f) -> typed function\n" "\n" @@ -376,1298 +36,6 @@ PyDoc_STRVAR(Function_doc, class Function : public Type { public: - class FunctionArg { - public: - FunctionArg(std::string name, Type* typeFilterOrNull, PyObject* defaultValue, bool isStarArg, bool isKwarg) : - m_name(name), - m_typeFilter(typeFilterOrNull), - m_defaultValue(defaultValue), - m_isStarArg(isStarArg), - m_isKwarg(isKwarg) - { - assert(!(isStarArg && isKwarg)); - } - - std::string getName() const { - return m_name; - } - - PyObject* getDefaultValue() const { - return m_defaultValue; - } - - Type* getTypeFilter() const { - return m_typeFilter; - } - - bool getIsStarArg() const { - return m_isStarArg; - } - - bool getIsKwarg() const { - return m_isKwarg; - } - - bool getIsNormalArg() const { - return !m_isKwarg && !m_isStarArg; - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - if (m_typeFilter) { - visitor(m_typeFilter); - } - } - - bool operator<(const FunctionArg& other) const { - if (m_name < other.m_name) { - return true; - } - if (m_name > other.m_name) { - return false; - } - if (m_typeFilter < other.m_typeFilter) { - return true; - } - if (m_typeFilter > other.m_typeFilter) { - return false; - } - if (m_defaultValue < other.m_defaultValue) { - return true; - } - if (m_defaultValue > other.m_defaultValue) { - return false; - } - if (m_isStarArg < other.m_isStarArg) { - return true; - } - if (m_isStarArg > other.m_isStarArg) { - return false; - } - if (m_isKwarg < other.m_isKwarg) { - return true; - } - if (m_isKwarg > other.m_isKwarg) { - return false; - } - - return false; - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - - buffer.writeStringObject(0, m_name); - if (m_typeFilter) { - context.serializeNativeType(m_typeFilter, buffer, 1); - } - if (m_defaultValue) { - context.serializePythonObject(m_defaultValue, buffer, 2); - } - buffer.writeUnsignedVarintObject(3, m_isStarArg ? 1 : 0); - buffer.writeUnsignedVarintObject(4, m_isKwarg ? 1 : 0); - - buffer.writeEndCompound(); - } - - template - static FunctionArg deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - std::string name; - Type* typeFilterOrNull = nullptr; - PyObjectHolder defaultValue; - bool isStarArg = false; - bool isKwarg = false; - - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber == 0) { - assertWireTypesEqual(wireType, WireType::BYTES); - name = buffer.readStringObject(); - } - else if (fieldNumber == 1) { - typeFilterOrNull = context.deserializeNativeType(buffer, wireType); - } - else if (fieldNumber == 2) { - defaultValue.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 3) { - assertWireTypesEqual(wireType, WireType::VARINT); - isStarArg = buffer.readUnsignedVarint(); - } - else if (fieldNumber == 4) { - assertWireTypesEqual(wireType, WireType::VARINT); - isKwarg = buffer.readUnsignedVarint(); - } - }); - - return FunctionArg(name, typeFilterOrNull, defaultValue, isStarArg, isKwarg); - } - - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitName(m_name); - if (m_defaultValue) { - v.visitTopo((PyObject*)m_defaultValue); - } else { - v.visitHash(ShaHash()); - } - - if (m_typeFilter) { - v.visitTopo(m_typeFilter); - } else { - v.visitHash(ShaHash()); - } - - v.visitHash( - ShaHash((m_isStarArg ? 2 : 1) + (m_isKwarg ? 10: 11)) - ); - } - - private: - std::string m_name; - Type* m_typeFilter; - PyObjectHolder m_defaultValue; - bool m_isStarArg; - bool m_isKwarg; - }; - - class CompiledSpecialization { - public: - CompiledSpecialization( - compiled_code_entrypoint funcPtr, - Type* returnType, - const std::vector& argTypes - ) : - mFuncPtr(funcPtr), - mReturnType(returnType), - mArgTypes(argTypes) - {} - - compiled_code_entrypoint getFuncPtr() const { - return mFuncPtr; - } - - Type* getReturnType() const { - return mReturnType; - } - - const std::vector& getArgTypes() const { - return mArgTypes; - } - - bool operator==(const CompiledSpecialization& other) const { - return mFuncPtr == other.mFuncPtr - && mReturnType == other.mReturnType - && mArgTypes == other.mArgTypes - ; - } - - private: - compiled_code_entrypoint mFuncPtr; - Type* mReturnType; - std::vector mArgTypes; - }; - - class Overload { - public: - Overload( - PyObject* pyFuncCode, - PyObject* pyFuncGlobals, - PyObject* pyFuncDefaults, - PyObject* pyFuncAnnotations, - const std::map& pyFuncGlobalsInCells, - const std::vector& pyFuncClosureVarnames, - const std::map closureBindings, - Type* returnType, - PyObject* pySignatureFunction, - const std::vector& args, - Type* methodOf - ) : - mFunctionCode(pyFuncCode), - mFunctionGlobals(pyFuncGlobals), - mFunctionDefaults(pyFuncDefaults), - mFunctionAnnotations(pyFuncAnnotations), - mFunctionGlobalsInCells(pyFuncGlobalsInCells), - mFunctionClosureVarnames(pyFuncClosureVarnames), - mReturnType(returnType), - mSignatureFunction(pySignatureFunction), - mArgs(args), - mHasKwarg(false), - mHasStarArg(false), - mMinPositionalArgs(0), - mMaxPositionalArgs(-1), - mClosureBindings(closureBindings), - mCachedFunctionObj(nullptr), - mMethodOf(methodOf) - { - long argsWithDefaults = 0; - long argsDefinitelyConsuming = 0; - - for (auto arg: mArgs) { - if (arg.getIsStarArg()) { - mHasStarArg = true; - } - else if (arg.getIsKwarg()) { - mHasKwarg = true; - } - else if (arg.getDefaultValue()) { - argsWithDefaults++; - } else { - argsDefinitelyConsuming++; - } - } - - mMinPositionalArgs = argsDefinitelyConsuming; - if (!mHasStarArg) { - mMaxPositionalArgs = argsDefinitelyConsuming + argsWithDefaults; - } - - increfAllPyObjects(); - } - - Overload(const Overload& other) { - other.increfAllPyObjects(); - - mFunctionCode = other.mFunctionCode; - mFunctionGlobals = other.mFunctionGlobals; - mFunctionDefaults = other.mFunctionDefaults; - mFunctionAnnotations = other.mFunctionAnnotations; - mSignatureFunction = other.mSignatureFunction; - mMethodOf = other.mMethodOf; - - mFunctionClosureVarnames = other.mFunctionClosureVarnames; - - mClosureBindings = other.mClosureBindings; - mReturnType = other.mReturnType; - mArgs = other.mArgs; - mCompiledSpecializations = other.mCompiledSpecializations; - - mHasStarArg = other.mHasStarArg; - mHasKwarg = other.mHasKwarg; - mMinPositionalArgs = other.mMinPositionalArgs; - mMaxPositionalArgs = other.mMaxPositionalArgs; - - mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; - - mCachedFunctionObj = other.mCachedFunctionObj; - } - - ~Overload() { - decrefAllPyObjects(); - } - - Overload withShiftedFrontClosureBindings(long shiftAmount) const { - std::map bindings; - for (auto nameAndBinding: mClosureBindings) { - bindings[nameAndBinding.first] = nameAndBinding.second.withShiftedFrontBinding(shiftAmount); - } - - return Overload( - mFunctionCode, - mFunctionGlobals, - mFunctionDefaults, - mFunctionAnnotations, - mFunctionGlobalsInCells, - mFunctionClosureVarnames, - bindings, - mReturnType, - mSignatureFunction, - mArgs, - mMethodOf - ); - } - - Overload withMethodOf(Type* methodOf) const { - return Overload( - mFunctionCode, - mFunctionGlobals, - mFunctionDefaults, - mFunctionAnnotations, - mFunctionGlobalsInCells, - mFunctionClosureVarnames, - mClosureBindings, - mReturnType, - mSignatureFunction, - mArgs, - methodOf - ); - } - - Overload withClosureBindings(const std::map &bindings) const { - return Overload( - mFunctionCode, - mFunctionGlobals, - mFunctionDefaults, - mFunctionAnnotations, - mFunctionGlobalsInCells, - mFunctionClosureVarnames, - bindings, - mReturnType, - mSignatureFunction, - mArgs, - mMethodOf - ); - } - - std::string toString() const { - std::ostringstream str; - - str << "("; - - for (long k = 0; k < mArgs.size(); k++) { - if (k) { - str << ", "; - } - - if (mArgs[k].getIsStarArg()) { - str << "*"; - } - - if (mArgs[k].getIsKwarg()) { - str << "**"; - } - - str << mArgs[k].getName(); - - if (mArgs[k].getDefaultValue()) { - str << "=..."; - } - - if (mArgs[k].getTypeFilter()) { - str << ": " << mArgs[k].getTypeFilter()->name(); - } - } - - str << ")"; - - if (mReturnType) { - str << " -> " << mReturnType->name(); - } - - return str.str(); - } - - // return the FunctionArg* that a positional argument would map to, or 'nullptr' if - // it wouldn't - const FunctionArg* argForPositionalArgument(long argIx) const { - if (argIx >= mArgs.size()) { - return nullptr; - } - - if (mArgs[argIx].getIsStarArg() || mArgs[argIx].getIsKwarg()) { - return nullptr; - } - - return &mArgs[argIx]; - } - - // can we possibly match 'argCount' positional arguments? - bool couldMatchPositionalCount(long argCount) const { - return argCount >= mMinPositionalArgs && argCount < mMaxPositionalArgs; - } - - Type* getReturnType() const { - return mReturnType; - } - - const std::vector& getFunctionClosureVarnames() const { - return mFunctionClosureVarnames; - } - - const std::vector& getArgs() const { - return mArgs; - } - - void finalizeTypeConcrete() { - // ensure that the Function object we represent is in sync with - // our actual return and argument types. It would be better to not have - // any direct references to the python interpreter in this class and - // rebuild the concrete function object on demand later... - if (mReturnType) { - if (mFunctionAnnotations - && PyDict_Check(mFunctionAnnotations) - ) { - PyDict_SetItemString( - mFunctionAnnotations, - "return", - (PyObject*)PyInstance::typeObj(mReturnType) - ); - } - } - for (auto& a: mArgs) { - Type* typeFilter = a.getTypeFilter(); - - if (typeFilter - && mFunctionAnnotations - && PyDict_Check(mFunctionAnnotations) - ) { - PyDict_SetItemString( - mFunctionAnnotations, - a.getName().c_str(), - (PyObject*)PyInstance::typeObj(a.getTypeFilter()) - ); - } - } - } - - template - void _visitReferencedTypes(const visitor_type& visitor) { - // we need to keep mArgs and mReturnType in sync with - // mAnnotations, so if we change one of our types we - // need to update the resulting dictionary as well. - if (mReturnType) { - visitor(mReturnType); - } - for (auto& a: mArgs) { - a._visitReferencedTypes(visitor); - } - - for (auto& varnameAndBinding: mClosureBindings) { - varnameAndBinding.second._visitReferencedTypes(visitor); - } - - if (mMethodOf) { - visitor(mMethodOf); - } - - for (auto nameAndGlobal: mFunctionGlobalsInCells) { - if (!PyCell_Check(nameAndGlobal.second)) { - throw std::runtime_error( - "A global in mFunctionGlobalsInCells is somehow not a cell" - ); - } - - PyObject* cellContents = PyCell_Get(nameAndGlobal.second); - - if (cellContents && PyType_Check(cellContents)) { - Type* t = PyInstance::extractTypeFrom(cellContents); - if (t) { - visitor(t); - } - } - } - - _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - if (PyType_Check(val)) { - Type* t = PyInstance::extractTypeFrom(val); - if (t) { - visitor(t); - } - } - }); - } - - template - void _visitCompilerVisibleInternals(const visitor_type& visitor) { - visitor.visitTopo(mFunctionCode); - - if (mFunctionAnnotations) { - visitor.visitHash(ShaHash(2)); - - if (PyDict_CheckExact(mFunctionAnnotations)) { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(mFunctionAnnotations, &pos, &key, &value)) { - visitor.visitTopo(key); - visitor.visitTopo(value); - } - } - - visitor.visitHash(ShaHash(2)); - } - - if (mSignatureFunction) { - visitor.visitHash(ShaHash(1)); - visitor.visitTopo(mSignatureFunction); - } else { - visitor.visitHash(ShaHash(0)); - } - - if (mFunctionDefaults) { - visitor.visitHash(ShaHash(2)); - - if (PyDict_CheckExact(mFunctionDefaults)) { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(mFunctionDefaults, &pos, &key, &value)) { - visitor.visitTopo(key); - visitor.visitTopo(value); - } - } - - visitor.visitHash(ShaHash(2)); - } - - visitor.visitHash(ShaHash(mFunctionGlobalsInCells.size())); - - for (auto nameAndGlobal: mFunctionGlobalsInCells) { - if (!PyCell_Check(nameAndGlobal.second)) { - throw std::runtime_error( - "A global in mFunctionGlobalsInCells is somehow not a cell" - ); - } - - visitor.visitNamedTopo( - nameAndGlobal.first, - nameAndGlobal.second - ); - } - - if (mReturnType) { - visitor.visitTopo(mReturnType); - } else { - visitor.visitHash(ShaHash()); - } - - if (mMethodOf) { - visitor.visitTopo(mMethodOf); - } else { - visitor.visitHash(ShaHash()); - } - - visitor.visitHash(ShaHash(mClosureBindings.size())); - for (auto nameAndClosure: mClosureBindings) { - visitor.visitName(nameAndClosure.first); - nameAndClosure.second._visitCompilerVisibleInternals(visitor); - } - - visitor.visitHash(ShaHash(mArgs.size())); - - for (auto a: mArgs) { - a._visitCompilerVisibleInternals(visitor); - } - - visitor.visitHash(ShaHash(1)); - - _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - visitor.visitNamedTopo(name, val); - }); - - visitor.visitHash(ShaHash(1)); - } - - template - static void visitCompilerVisibleGlobals( - const visitor_type& visitor, - PyCodeObject* code, - PyObject* globals - ) { - std::vector > dotAccesses; - - extractDottedGlobalAccessesFromCode(code, dotAccesses); - - auto visitSequence = [&](const std::vector& sequence) { - PyObjectHolder curObj; - std::string curName; - - for (PyObject* name: sequence) { - std::string nameStr = PyUnicode_AsUTF8(name); - - if (isSpecialIgnorableName(nameStr)) { - break; - } - - if (!curObj) { - curName = nameStr; - } else { - curName = curName + "." + nameStr; - } - - if (!curObj) { - // this is a lookup in the global dict - curObj.set(PyDict_GetItem(globals, name)); - - if (!curObj) { - // this is an invalid global lookup, which is OK. no need to hash anything. - PyErr_Clear(); - return; - } - } else { - // we're looking up an attribute of this object. We only want to look into modules. - if (PyModule_CheckExact(curObj) && PyObject_HasAttr(curObj, name)) { - PyObjectStealer moduleMember(PyObject_GetAttr(curObj, name)); - - curObj.steal(PyObject_GetAttr(curObj, name)); - if (!curObj) { - // this is an invalid module member lookup. We can just bail. - PyErr_Clear(); - return; - } - } else { - break; - } - } - } - - // also visit at the end of the sequence - if (curObj) { - visitor(curName, (PyObject*)curObj); - } - }; - - for (auto& sequence: dotAccesses) { - visitSequence(sequence); - } - } - - template - void _visitCompilerVisibleGlobals(const visitor_type& visitor) { - visitCompilerVisibleGlobals(visitor, (PyCodeObject*)mFunctionCode, mFunctionGlobals); - } - - template - void _visitContainedTypes(const visitor_type& visitor) { - } - - const std::vector& getCompiledSpecializations() const { - return mCompiledSpecializations; - } - - void addCompiledSpecialization(compiled_code_entrypoint e, Type* returnType, const std::vector& argTypes) { - CompiledSpecialization newSpec = CompiledSpecialization(e,returnType,argTypes); - - for (auto& spec: mCompiledSpecializations) { - if (spec == newSpec) { - return; - } - } - - mCompiledSpecializations.push_back(newSpec); - } - - void touchCompiledSpecializations() { - //force the memory for the compiled specializations to move. - std::vector other = mCompiledSpecializations; - std::swap(mCompiledSpecializations, other); - } - - bool operator<(const Overload& other) const { - if (mFunctionCode < other.mFunctionCode) { return true; } - if (mFunctionCode > other.mFunctionCode) { return false; } - - if (mFunctionGlobals < other.mFunctionGlobals) { return true; } - if (mFunctionGlobals > other.mFunctionGlobals) { return false; } - - if (mFunctionGlobalsInCells < other.mFunctionGlobalsInCells) { return true; } - if (mFunctionGlobalsInCells > other.mFunctionGlobalsInCells) { return false; } - - if (mClosureBindings < other.mClosureBindings) { return true; } - if (mClosureBindings > other.mClosureBindings) { return false; } - - if (mReturnType < other.mReturnType) { return true; } - if (mReturnType > other.mReturnType) { return false; } - - if (mArgs < other.mArgs) { return true; } - if (mArgs > other.mArgs) { return false; } - - if (mMethodOf > other.mMethodOf) { return true; } - if (mMethodOf < other.mMethodOf) { return false; } - - return false; - } - - const std::map& getClosureVariableBindings() const { - return mClosureBindings; - } - - PyObject* getFunctionCode() const { - return mFunctionCode; - } - - PyObject* getFunctionDefaults() const { - return mFunctionDefaults; - } - - PyObject* getFunctionAnnotations() const { - return mFunctionAnnotations; - } - - Type* getMethodOf() const { - return mMethodOf; - } - - PyObject* getSignatureFunction() const { - return mSignatureFunction; - } - - PyObject* getFunctionGlobals() const { - return mFunctionGlobals; - } - - const std::map& getFunctionGlobalsInCells() const { - return mFunctionGlobalsInCells; - } - - /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. - - In cases where we write something like 'x.y.z' the compiler shouldn't have a reference to 'x', - just to whatever 'x.y.z' refers to. - - This transformation just figures out what the dotting sequences are. - */ - static void extractDottedGlobalAccessesFromCode(PyCodeObject* code, std::vector >& outSequences) { - uint8_t* bytes; - Py_ssize_t bytecount; - - static PyObject* moduleHashName = PyUnicode_FromString("__module_hash__"); - outSequences.push_back(std::vector({moduleHashName})); - - PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); - - long opcodeCount = bytecount / 2; - - // opcodes are encoded in the low byte - auto opcodeFor = [&](int i) { return bytes[i * 2]; }; - - // opcode targets are encoded in the high byte - auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; - - const uint8_t LOAD_ATTR = 106; - const uint8_t LOAD_GLOBAL = 116; - const uint8_t DELETE_GLOBAL = 98; - const uint8_t STORE_GLOBAL = 97; - const uint8_t LOAD_METHOD = 160; - - - std::vector curDotSequence; - for (long ix = 0; ix < opcodeCount; ix++) { - // if we're loading an attr on an existing sequence, just make it bigger - if ((opcodeFor(ix) == LOAD_ATTR || opcodeFor(ix) == LOAD_METHOD) && curDotSequence.size()) { - curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); - } else if (curDotSequence.size()) { - // any other operation should flush the buffer - outSequences.push_back(curDotSequence); - curDotSequence.clear(); - } - - // if we're loading a global, we start a new sequence - if (opcodeFor(ix) == LOAD_GLOBAL) { - curDotSequence.push_back(PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))); - } else if ( - opcodeFor(ix) == STORE_GLOBAL - || opcodeFor(ix) == DELETE_GLOBAL - ) { - outSequences.push_back({PyTuple_GetItem(code->co_names, opcodeTargetFor(ix))}); - } - } - - // flush the buffer if we have something - if (curDotSequence.size()) { - outSequences.push_back(curDotSequence); - } - - // recurse into sub code objects - iterate(code->co_consts, [&](PyObject* o) { - if (PyCode_Check(o)) { - extractDottedGlobalAccessesFromCode((PyCodeObject*)o, outSequences); - } - }); - } - - static void extractGlobalAccessesFromCode(PyCodeObject* code, std::set& outAccesses) { - uint8_t* bytes; - Py_ssize_t bytecount; - - PyBytes_AsStringAndSize(((PyCodeObject*)code)->co_code, (char**)&bytes, &bytecount); - - long opcodeCount = bytecount / 2; - - // opcodes are encoded in the low byte - auto opcodeFor = [&](int i) { return bytes[i * 2]; }; - - // opcode targets are encoded in the high byte - auto opcodeTargetFor = [&](int i) { return bytes[i * 2 + 1]; }; - - const uint8_t LOAD_GLOBAL = 116; - const uint8_t DELETE_GLOBAL = 98; - const uint8_t STORE_GLOBAL = 97; - - for (long ix = 0; ix < opcodeCount; ix++) { - // if we're loading a global, we start a new sequence - if (opcodeFor(ix) == LOAD_GLOBAL) { - PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); - if (!PyUnicode_Check(name)) { - throw std::runtime_error("Function had a non-string object in co_names"); - } - outAccesses.insert(PyUnicode_AsUTF8(name)); - } else if ( - opcodeFor(ix) == STORE_GLOBAL - || opcodeFor(ix) == DELETE_GLOBAL - ) { - PyObject* name = PyTuple_GetItem(code->co_names, opcodeTargetFor(ix)); - if (!PyUnicode_Check(name)) { - throw std::runtime_error("Function had a non-string object in co_names"); - } - outAccesses.insert(PyUnicode_AsUTF8(name)); - } - } - - // recurse into sub code objects - iterate(code->co_consts, [&](PyObject* o) { - if (PyCode_Check(o)) { - extractGlobalAccessesFromCode((PyCodeObject*)o, outAccesses); - } - }); - - outAccesses.insert("__module_hash__"); - } - - void extractGlobalAccessesFromCodeIncludingCells( - std::set& outNames - ) const { - for (auto nameAndGlobal: mFunctionGlobalsInCells) { - outNames.insert(nameAndGlobal.first); - } - - outNames.insert("__module_hash__"); - - extractGlobalAccessesFromCode((PyCodeObject*)mFunctionCode, outNames); - } - - void setGlobals(PyObject* globals) { - decref(mFunctionGlobals); - mFunctionGlobals = incref(globals); - } - - bool symbolIsUnresolved(std::string name, bool insistResolved) { - if (mClosureBindings.find(name) != mClosureBindings.end()) { - return false; - } - - auto it = mFunctionGlobalsInCells.find(name); - if (it != mFunctionGlobalsInCells.end()) { - if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - if (insistResolved) { - PyObject* o = PyCell_Get(it->second); - Type* t = PyInstance::extractTypeFrom(o); - if (t && t->isForwardDefined()) { - return true; - } - } - - return false; - } - - return true; - } - - if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { - if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { - if (insistResolved) { - PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - Type* t = PyInstance::extractTypeFrom(o); - if (t && t->isForwardDefined()) { - return true; - } - } - - return false; - } - - PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); - if (builtins && PyDict_GetItemString(builtins, name.c_str())) { - return false; - } - } - - return true; - } - - void autoresolveGlobal( - std::string name, - const std::set& resolvedForwards - ) { - auto it = mFunctionGlobalsInCells.find(name); - if (it != mFunctionGlobalsInCells.end()) { - if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - if (PyType_Check(PyCell_GET(it->second))) { - Type* cellContents = PyInstance::extractTypeFrom( - (PyTypeObject*)PyCell_Get(it->second) - ); - - if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - PyCell_Set( - it->second, - (PyObject*)PyInstance::typeObj( - cellContents->forwardResolvesTo() - ) - ); - } - } - } - - return; - } - - PyObject* dictVal = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - if (dictVal && PyType_Check(dictVal)) { - Type* cellContents = PyInstance::extractTypeFrom( - (PyTypeObject*)dictVal - ); - - if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - PyDict_SetItemString( - mFunctionGlobals, - name.c_str(), - (PyObject*)PyInstance::typeObj( - cellContents->forwardResolvesTo() - ) - ); - } - } - } - - void autoresolveGlobals(const std::set& resolvedForwards) { - std::set allNames; - extractGlobalAccessesFromCodeIncludingCells(allNames); - - for (auto nameStr: allNames) { - if (nameStr != "__module_hash__") { - autoresolveGlobal(nameStr, resolvedForwards); - } - } - } - - bool hasUnresolvedSymbols(bool insistResolved) { - std::set allNames; - extractGlobalAccessesFromCodeIncludingCells(allNames); - - for (auto nameStr: allNames) { - if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { - return true; - } - } - - return false; - } - - std::string firstUnresolvedSymbol(bool insistResolved) { - std::set allNames; - extractGlobalAccessesFromCodeIncludingCells(allNames); - - for (auto nameStr: allNames) { - if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { - return nameStr; - } - } - - return ""; - } - - PyObject* getUsedGlobals() const { - // restrict the globals to contain only the values it references. - PyObject* result = PyDict_New(); - - std::set allNamesString; - extractGlobalAccessesFromCodeIncludingCells(allNamesString); - - for (auto nameStr: allNamesString) { - if (nameStr != "__module_hash__" && mClosureBindings.find(nameStr) != mClosureBindings.end()) { - throw std::runtime_error( - "Somehow we have both a closure binding and a global for " - + nameStr - ); - } - } - - // iterate mFunctionGlobals, keeping any where the name is in allNames - // note we split on '.' and take the first part so that if a module - // like lxml.etree is included, and we use 'lxml', we'll take the reference - // to etree as well. This ensures that submodules in anonymously serialized - // code can get pulled along. - PyObject *key, *value; - Py_ssize_t pos = 0; - - // place them in sorted order - std::map > toCopy; - - while (PyDict_Next(mFunctionGlobals, &pos, &key, &value)) { - if (PyUnicode_Check(key)) { - std::string globalName = PyUnicode_AsUTF8(key); - std::string shortGlobalName = globalName; - - size_t indexOfDot = shortGlobalName.find('.'); - if (indexOfDot != std::string::npos) { - shortGlobalName = shortGlobalName.substr(0, indexOfDot); - } - - if (allNamesString.find(shortGlobalName) != allNamesString.end()) { - toCopy[globalName] = std::make_pair(key, value); - } - } - } - - for (auto& nameandKV: toCopy) { - if (PyDict_SetItem(result, nameandKV.second.first, nameandKV.second.second)) { - throw PythonExceptionSet(); - } - } - - PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); - if (builtins) { - PyDict_SetItemString(result, "__builtins__", builtins); - } - - return result; - } - - // create a new function object for this closure (or cache it - // if we have no closure) - PyObject* buildFunctionObj(Type* closureType, instance_ptr closureData) const; - - Overload& operator=(const Overload& other) { - other.increfAllPyObjects(); - decrefAllPyObjects(); - - mFunctionCode = other.mFunctionCode; - mFunctionGlobals = other.mFunctionGlobals; - mFunctionDefaults = other.mFunctionDefaults; - mFunctionAnnotations = other.mFunctionAnnotations; - mSignatureFunction = other.mSignatureFunction; - - mMethodOf = other.mMethodOf; - - mFunctionClosureVarnames = other.mFunctionClosureVarnames; - - mClosureBindings = other.mClosureBindings; - mReturnType = other.mReturnType; - mArgs = other.mArgs; - mCompiledSpecializations = other.mCompiledSpecializations; - - mHasStarArg = other.mHasStarArg; - mHasKwarg = other.mHasKwarg; - mMinPositionalArgs = other.mMinPositionalArgs; - mMaxPositionalArgs = other.mMaxPositionalArgs; - mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; - - mCachedFunctionObj = other.mCachedFunctionObj; - - return *this; - } - - template - void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - buffer.writeBeginCompound(fieldNumber); - - context.serializePythonObject(mFunctionCode, buffer, 0); - - if (mFunctionDefaults) { - context.serializePythonObject(mFunctionDefaults, buffer, 2); - } - - if (mFunctionAnnotations) { - context.serializePythonObject(mFunctionAnnotations, buffer, 3); - } - - buffer.writeBeginCompound(4); - int stringIx = 0; - for (auto varname: mFunctionClosureVarnames) { - buffer.writeStringObject(stringIx++, varname); - } - buffer.writeEndCompound(); - - buffer.writeBeginCompound(5); - int varIx = 0; - for (auto nameAndCell: mFunctionGlobalsInCells) { - buffer.writeStringObject(varIx++, nameAndCell.first); - context.serializePythonObject(nameAndCell.second, buffer, varIx++); - } - buffer.writeEndCompound(); - - buffer.writeBeginCompound(6); - int closureBindingIx = 0; - for (auto nameAndBinding: mClosureBindings) { - buffer.writeStringObject(closureBindingIx++, nameAndBinding.first); - nameAndBinding.second.serialize(context, buffer, closureBindingIx++); - } - buffer.writeEndCompound(); - - if (mReturnType) { - context.serializeNativeType(mReturnType, buffer, 7); - } - - buffer.writeBeginCompound(8); - - int argIx = 0; - for (auto& arg: mArgs) { - arg.serialize(context, buffer, argIx++); - } - - buffer.writeEndCompound(); - - if (mSignatureFunction) { - context.serializePythonObject(mSignatureFunction, buffer, 9); - } - - if (mMethodOf) { - context.serializeNativeType(mMethodOf, buffer, 10); - } - - buffer.writeEndCompound(); - } - - template - static Overload deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - PyObjectHolder functionCode; - PyObjectHolder functionGlobals; - PyObjectHolder functionAnnotations; - PyObjectHolder functionDefaults; - PyObjectHolder functionSignature; - std::vector closureVarnames; - std::map functionGlobalsInCells; - std::map functionGlobalsInCellsRaw; - std::map closureBindings; - Type* returnType = nullptr; - Type* methodOf = nullptr; - std::vector args; - - functionGlobals.steal(PyDict_New()); - - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber == 0) { - functionCode.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 2) { - functionDefaults.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 3) { - functionAnnotations.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 4) { - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - assertWireTypesEqual(wireType, WireType::BYTES); - closureVarnames.push_back(buffer.readStringObject()); - }); - } - else if (fieldNumber == 5) { - std::string last; - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber % 2 == 0) { - assertWireTypesEqual(wireType, WireType::BYTES); - last = buffer.readStringObject(); - } else { - if (last == "") { - throw std::runtime_error("Corrupt Function closure encountered"); - } - functionGlobalsInCells[last].steal(context.deserializePythonObject(buffer, wireType)); - functionGlobalsInCellsRaw[last] = functionGlobalsInCells[last]; - last = ""; - } - }); - } - else if (fieldNumber == 6) { - std::string last; - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - if (fieldNumber % 2 == 0) { - assertWireTypesEqual(wireType, WireType::BYTES); - last = buffer.readStringObject(); - } else { - closureBindings[last] = ClosureVariableBinding::deserialize(context, buffer, wireType); - } - }); - } - else if (fieldNumber == 7) { - returnType = context.deserializeNativeType(buffer, wireType); - } - else if (fieldNumber == 8) { - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - args.push_back(FunctionArg::deserialize(context, buffer, wireType)); - }); - } - else if (fieldNumber == 9) { - functionSignature.steal(context.deserializePythonObject(buffer, wireType)); - } - else if (fieldNumber == 10) { - methodOf = context.deserializeNativeType(buffer, wireType); - } - }); - - return Overload( - functionCode, - functionGlobals, - functionDefaults, - functionAnnotations, - functionGlobalsInCellsRaw, - closureVarnames, - closureBindings, - returnType, - functionSignature, - args, - methodOf - ); - } - - void increfAllPyObjects() const { - incref(mFunctionCode); - incref(mFunctionGlobals); - incref(mFunctionDefaults); - incref(mFunctionAnnotations); - incref(mSignatureFunction); - - for (auto nameAndOther: mFunctionGlobalsInCells) { - incref(nameAndOther.second); - } - } - - void decrefAllPyObjects() { - decref(mFunctionCode); - decref(mFunctionGlobals); - decref(mFunctionDefaults); - decref(mFunctionAnnotations); - decref(mSignatureFunction); - - for (auto nameAndOther: mFunctionGlobalsInCells) { - decref(nameAndOther.second); - } - } - - private: - PyObject* mFunctionCode; - - PyObject* mFunctionGlobals; - - // globals that are stored in cells. This happens when class objects - // are defined inside of function scopes. We assume that anything in their - // closure is global (and therefore constant) but it may not be defined yet, - // so we can't just pull the value out and stick it in the function closure - // itself. Each value in the map is guaranteed to be a 'cell' object. - std::map mFunctionGlobalsInCells; - - // the order (by name) of the variables in the __closure__ of the original - // function. This is the order that the python code will expect. - std::vector mFunctionClosureVarnames; - - PyObject* mFunctionDefaults; - - PyObject* mFunctionAnnotations; - - PyObject* mSignatureFunction; - - // note that we are deliberately leaking this value because Overloads get - // stashed in static std::map memos anyways. - mutable PyObject* mCachedFunctionObj; - - std::map mClosureBindings; - - Type* mReturnType; - - // if we are a method of a class, what class? Used to - Type* mMethodOf; - - std::vector mArgs; - - // in compiled code, the closure arguments get passed in front of the - // actual function arguments - std::vector mCompiledSpecializations; - - bool mHasStarArg; - bool mHasKwarg; - size_t mMinPositionalArgs; - size_t mMaxPositionalArgs; - }; - Function() : Type(catFunction) { } @@ -1679,7 +47,7 @@ class Function : public Type { Function(std::string inName, std::string qualname, std::string moduleName, - const std::vector& overloads, + const std::vector& overloads, Type* closureType, bool isEntrypoint, bool isNocompile @@ -1701,7 +69,7 @@ class Function : public Type { std::string inName, std::string qualname, std::string moduleName, - std::vector overloads, + std::vector overloads, Type* closureType, bool isEntrypoint, bool isNocompile @@ -1821,7 +189,7 @@ class Function : public Type { std::string inName, std::string qualname, std::string moduleName, - std::vector overloads, + std::vector overloads, Type* closureType, bool isEntrypoint, bool isNocompile @@ -1856,7 +224,7 @@ class Function : public Type { const std::string, const std::string, const std::string, - const std::vector, + const std::vector, Type*, bool, bool @@ -1910,7 +278,7 @@ class Function : public Type { types.push_back(t); } - std::vector overloads(f1->mOverloads); + std::vector overloads(f1->mOverloads); for (auto o: f2->mOverloads) { overloads.push_back(o.withShiftedFrontClosureBindings(((Tuple*)f1->getClosureType())->getTypes().size())); } @@ -2007,7 +375,7 @@ class Function : public Type { return mClosureType->isPOD(); } - const std::vector& getOverloads() const { + const std::vector& getOverloads() const { return mOverloads; } @@ -2055,7 +423,7 @@ class Function : public Type { return this; } - std::vector overloads; + std::vector overloads; for (auto& o: mOverloads) { overloads.push_back(o.withMethodOf(methodOf)); } @@ -2136,12 +504,12 @@ class Function : public Type { return Function::Make(mRootName, mQualname, mModulename, mOverloads, closureType, mIsEntrypoint, mIsNocompile); } - Function* replaceOverloads(const std::vector& overloads) const { + Function* replaceOverloads(const std::vector& overloads) const { return Function::Make(mRootName, mQualname, mModulename, overloads, mClosureType, mIsEntrypoint, mIsNocompile); } Function* replaceOverloadVariableBindings(long index, const std::map& bindings) { - std::vector overloads(mOverloads); + std::vector overloads(mOverloads); if (index < 0 || index >= mOverloads.size()) { throw std::runtime_error("Invalid index to replaceOverloadVariableBindings"); } @@ -2164,7 +532,7 @@ class Function : public Type { } private: - std::vector mOverloads; + std::vector mOverloads; Type* mClosureType; diff --git a/typed_python/ModuleRepresentationCopyContext.hpp b/typed_python/ModuleRepresentationCopyContext.hpp index 5a102fdbe..3e8005800 100644 --- a/typed_python/ModuleRepresentationCopyContext.hpp +++ b/typed_python/ModuleRepresentationCopyContext.hpp @@ -207,13 +207,13 @@ class ModuleRepresentationCopyContext { Function* f = (Function*)t; - std::vector overloads; + std::vector overloads; for (auto& o: f->getOverloads()) { - std::vector args; + std::vector args; for (auto& a: o.getArgs()) { args.push_back( - Function::FunctionArg( + FunctionArg( a.getName(), copyType(a.getTypeFilter()), copyObj(a.getDefaultValue()), @@ -224,7 +224,7 @@ class ModuleRepresentationCopyContext { } overloads.push_back( - Function::Overload( + FunctionOverload( o.getFunctionCode(), copyObj(o.getFunctionGlobals()), copyObj(o.getFunctionDefaults()), diff --git a/typed_python/PyFunctionInstance.cpp b/typed_python/PyFunctionInstance.cpp index 3ca04ed12..6ecaa5aac 100644 --- a/typed_python/PyFunctionInstance.cpp +++ b/typed_python/PyFunctionInstance.cpp @@ -105,7 +105,7 @@ std::pair PyFunctionInstance::tryToCallOverload( PyObject* kwargs, ConversionLevel conversionLevel ) { - const Function::Overload& overload(f->getOverloads()[overloadIx]); + const FunctionOverload& overload(f->getOverloads()[overloadIx]); FunctionCallArgMapping mapping(overload); @@ -196,7 +196,7 @@ std::pair PyFunctionInstance::getOverloadReturnType( long overloadIx, FunctionCallArgMapping& matchedArgs ) { - const Function::Overload& overload(f->getOverloads()[overloadIx]); + const FunctionOverload& overload(f->getOverloads()[overloadIx]); Type* returnType = overload.getReturnType(); if (overload.getSignatureFunction()) { @@ -376,7 +376,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( long overloadIx, const FunctionCallArgMapping& mapper ) { - const Function::Overload& overload(f->getOverloads()[overloadIx]); + const FunctionOverload& overload(f->getOverloads()[overloadIx]); for (const auto& spec: overload.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -420,7 +420,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( throw std::runtime_error("Somehow, the number of overloads in the function changed."); } - const Function::Overload& overload2(convertedF->getOverloads()[overloadIx]); + const FunctionOverload& overload2(convertedF->getOverloads()[overloadIx]); for (const auto& spec: overload2.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -457,7 +457,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( decref(res); - const Function::Overload& convertedOverload(convertedF->getOverloads()[overloadIx]); + const FunctionOverload& convertedOverload(convertedF->getOverloads()[overloadIx]); for (const auto& spec: convertedOverload.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -484,10 +484,10 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( } std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpecialization( - const Function::Overload& overload, + const FunctionOverload& overload, Type* closureType, instance_ptr closureData, - const Function::CompiledSpecialization& specialization, + const CompiledSpecialization& specialization, const FunctionCallArgMapping& mapper ) { Type* returnType = specialization.getReturnType(); @@ -496,7 +496,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpe throw std::runtime_error("Malformed function specialization: missing a return type."); } - std::vector mappingArgs; + std::vector mappingArgs; // first, see if we can short-circuit for (long k = 0; k < overload.getArgs().size(); k++) { @@ -514,7 +514,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpe auto arg = overload.getArgs()[k]; Type* argType = specialization.getArgTypes()[k]; - FunctionCallArgMapping::FunctionArg res = mapper.extractArgWithType(k, argType); + FunctionCallArgMapping::LiveFunctionArg res = mapper.extractArgWithType(k, argType); if (res.isValid()) { mappingArgs.push_back(res); diff --git a/typed_python/PyFunctionInstance.hpp b/typed_python/PyFunctionInstance.hpp index d7a9deab7..5f56fe796 100644 --- a/typed_python/PyFunctionInstance.hpp +++ b/typed_python/PyFunctionInstance.hpp @@ -84,10 +84,10 @@ class PyFunctionInstance : public PyInstance { //we can't convert, then return . If we do dispatch, return and set //the python exception if native code returns an exception. static std::pair dispatchFunctionCallToCompiledSpecialization( - const Function::Overload& overload, + const FunctionOverload& overload, Type* closureType, instance_ptr closureData, - const Function::CompiledSpecialization& specialization, + const CompiledSpecialization& specialization, const FunctionCallArgMapping& mapping ); diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 378da3ca8..bf769aff5 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1066,7 +1066,7 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur ); } - ((Function::Overload*)&f->getOverloads()[overloadIx])->setGlobals(overloadGlobals); + ((FunctionOverload*)&f->getOverloads()[overloadIx])->setGlobals(overloadGlobals); } }); }); @@ -1224,7 +1224,7 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur std::cout << "finalizing group:\n"; std::cout << outGroup->repr() << "\n"; - + outGroup->finalizeDeserializerGroup(); } @@ -1542,7 +1542,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( PyObjectHolder obj; Instance instance; - std::vector overloads; + std::vector overloads; Type* closureType = 0; int isEntrypoint = 0; int isNocompile = 0; @@ -1658,7 +1658,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( } else { overloads.push_back( - Function::Overload::deserialize(*this, b, wireType) + FunctionOverload::deserialize(*this, b, wireType) ); } } else diff --git a/typed_python/TypedClosureBuilder.hpp b/typed_python/TypedClosureBuilder.hpp index ddc76cdd8..6dd70846b 100644 --- a/typed_python/TypedClosureBuilder.hpp +++ b/typed_python/TypedClosureBuilder.hpp @@ -332,7 +332,7 @@ class TypedClosureBuilder { instance_ptr closureData = ((PyInstance*)(PyObject*)funcObj)->dataPtr(); Tuple* closureType = (Tuple*)funcT->getClosureType(); - std::vector newOverloads; + std::vector newOverloads; if (closureType->getTypes().size() != funcT->getOverloads().size()) { throw std::runtime_error("Untyped closures should have one element per overload."); @@ -354,7 +354,7 @@ class TypedClosureBuilder { return funcT->replaceClosure(mClosureType)->replaceOverloads(newOverloads); } - Function::Overload mapOverload(const Function::Overload& overload, NamedTuple* untypedClosure, instance_ptr untypedClosureData) { + FunctionOverload mapOverload(const FunctionOverload& overload, NamedTuple* untypedClosure, instance_ptr untypedClosureData) { std::map bindings; for (long cellIx = 0; cellIx < untypedClosure->getTypes().size(); cellIx++) { diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 9a1d46ced..57545fe0c 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -526,7 +526,7 @@ PyObject *getCodeGlobalDotAccesses(PyObject* nullValue, PyObject* args, PyObject std::vector > v; - Function::Overload::extractDottedGlobalAccessesFromCode( + FunctionOverload::extractDottedGlobalAccessesFromCode( (PyCodeObject*)pyCodeObject, v ); @@ -1000,7 +1000,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } - std::vector argList; + std::vector argList; for (long k = 0; k < PyTuple_Size(argTuple); k++) { PyObjectHolder kTup(PyTuple_GetItem(argTuple, k)); @@ -1048,7 +1048,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { incref(val); } - argList.push_back(Function::FunctionArg( + argList.push_back(FunctionArg( PyUnicode_AsUTF8(k0), argT, val, @@ -1073,7 +1073,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { decref(pyModulename); } - std::vector overloads; + std::vector overloads; std::vector closureVarnames; std::vector closureVarTypes; @@ -1130,7 +1130,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { } overloads.push_back( - Function::Overload( + FunctionOverload( PyFunction_GetCode(funcObj), PyFunction_GetGlobals(funcObj), PyFunction_GetDefaults(funcObj), diff --git a/typed_python/all.cpp b/typed_python/all.cpp index 76acff4ee..a3c69de52 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -60,7 +60,8 @@ compile the entire group all at once. #include "StringType.cpp" #include "TupleOrListOfType.cpp" #include "Type.cpp" -#include "FunctionType.cpp" +#include "ClosureVariableBinding.cpp" +#include "FunctionOverload.cpp" #include "ForwardType.cpp" #include "ValueType.cpp" From b5a22c90c6207c3bdb0e1e7fbf6247d238ac7126 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 16 Jun 2023 21:42:58 +0000 Subject: [PATCH 36/83] Fixing more serialization bugs. Type objects need to get 'finalized' in a separate pass after we construct them. --- repro.py | 46 +++++++++------- typed_python/FunctionOverload.hpp | 54 ++++++++++++++----- typed_python/FunctionType.hpp | 51 +++++++++--------- typed_python/PyInstance.cpp | 25 +++++---- typed_python/PyInstance.hpp | 2 +- ...onSerializationContext_deserialization.cpp | 19 +++---- ...thonSerializationContext_serialization.cpp | 2 +- typed_python/Type.cpp | 2 +- typed_python/Type.hpp | 4 +- typed_python/_types.cpp | 7 --- typed_python/types_serialization_test.py | 38 ++++++++++--- 11 files changed, 155 insertions(+), 95 deletions(-) diff --git a/repro.py b/repro.py index 52f365078..c378cd80a 100644 --- a/repro.py +++ b/repro.py @@ -1,36 +1,46 @@ import sys -from typed_python import Class, SerializationContext, Held, isForwardDefined +from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr - def writer(): - @Held - class C(Class): - def f(self): - return C + Base = Forward("Base") + + @Base.define + class Base(Class): + def blah(self) -> Base: + return self + + def f(self, x) -> int: + return x + 1 - assert not isForwardDefined(C) + class Child(Base, Final): + def f(self, x) -> int: + return -1 - print("group is:") - print(typeWalkRecord(type(C.f).overloads[0].funcGlobalsInCells['C'], 'identity')) - print(typeWalkRecord(type(C.f).overloads[0].funcGlobalsInCells['C'], 'compiler')) + aChild = Child() - # with open("a.dat", "wb") as f: - # f.write(SerializationContext().serialize(C)) + aChildBytes = SerializationContext().serialize(aChild) + + with open("a.dat", "wb") as f: + f.write(aChildBytes) def reader(): - with open("a.dat", "rb") as f: - C = SerializationContext().deserialize(f.read()) + with open("a.dat", "rb") as f: + aChild = SerializationContext().deserialize(f.read()) + + # @Entrypoint + def callF(x): + return x.f(10) + + assert callF(aChild) == 11 - print(C) - #assert C().f() is C if sys.argv[1:] == ['r']: - reader() + reader() else: - writer() + writer() diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 399dc0b7c..b54174640 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -368,10 +368,24 @@ class FunctionOverload { ); } - visitor.visitNamedTopo( - nameAndGlobal.first, - nameAndGlobal.second - ); + PyObject* o = PyCell_Get(nameAndGlobal.second); + Type* t = PyInstance::extractTypeFrom(o); + + if (t && t->isForwardDefined()) { + if (((Forward*)t)->isResolved()) { + visitor.visitNamedTopo( + nameAndGlobal.first, + ((Forward*)t)->forwardResolvesTo() + ); + } else { + // deliberately ignore non-resolved forwards + } + } else { + visitor.visitNamedTopo( + nameAndGlobal.first, + nameAndGlobal.second + ); + } } if (mReturnType) { @@ -401,7 +415,23 @@ class FunctionOverload { visitor.visitHash(ShaHash(1)); _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - visitor.visitNamedTopo(name, val); + Type* t = PyInstance::extractTypeFrom(val); + + if (t && t->isForwardDefined()) { + if (((Forward*)t)->isResolved()) { + visitor.visitNamedTopo( + name, + ((Forward*)t)->forwardResolvesTo() + ); + } else { + // deliberately ignore non-resolved forwards + } + } else { + visitor.visitNamedTopo( + name, + val + ); + } }); visitor.visitHash(ShaHash(1)); @@ -690,7 +720,7 @@ class FunctionOverload { mFunctionGlobals = incref(globals); } - bool symbolIsUnresolved(std::string name, bool insistResolved) { + bool symbolIsUnresolved(std::string name, bool insistForwardsResolved) { if (mClosureBindings.find(name) != mClosureBindings.end()) { return false; } @@ -698,7 +728,7 @@ class FunctionOverload { auto it = mFunctionGlobalsInCells.find(name); if (it != mFunctionGlobalsInCells.end()) { if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - if (insistResolved) { + if (insistForwardsResolved) { PyObject* o = PyCell_Get(it->second); Type* t = PyInstance::extractTypeFrom(o); if (t && t->isForwardDefined()) { @@ -714,7 +744,7 @@ class FunctionOverload { if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { - if (insistResolved) { + if (insistForwardsResolved) { PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); Type* t = PyInstance::extractTypeFrom(o); if (t && t->isForwardDefined()) { @@ -789,12 +819,12 @@ class FunctionOverload { } } - bool hasUnresolvedSymbols(bool insistResolved) { + bool hasUnresolvedSymbols(bool insistForwardsResolved) { std::set allNames; extractGlobalAccessesFromCodeIncludingCells(allNames); for (auto nameStr: allNames) { - if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistForwardsResolved)) { return true; } } @@ -802,12 +832,12 @@ class FunctionOverload { return false; } - std::string firstUnresolvedSymbol(bool insistResolved) { + std::string firstUnresolvedSymbol(bool insistForwardsResolved) { std::set allNames; extractGlobalAccessesFromCodeIncludingCells(allNames); for (auto nameStr: allNames) { - if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistResolved)) { + if (nameStr != "__module_hash__" && symbolIsUnresolved(nameStr, insistForwardsResolved)) { return nameStr; } } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 610f7f366..559b9f0d8 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -96,9 +96,10 @@ class Function : public Type { } // does this function have any of its globals that are not - // resolved to actual values. if 'insistResolved' then we - // also return 'true' if the symbols resolve to a forward defined - // type + // resolved to actual values. if 'insistForwardsResolved' then we + // return 'true' if the symbol resolves to a forward defined + // type - the symbol is only considered resolved when there are no + // visible forwards std::string firstUnresolvedSymbol(bool insistResolved) { for (auto& o: mOverloads) { std::string res = o.firstUnresolvedSymbol(insistResolved); @@ -163,28 +164,6 @@ class Function : public Type { }); } - template - void _visitCompilerVisibleInternals(const visitor_type& v) { - v.visitHash( - ShaHash(1, m_typeCategory) - + ShaHash(mIsNocompile ? 2 : 1) - + ShaHash(mIsEntrypoint ? 2 : 1) - ); - - v.visitName(m_name); - v.visitName(mRootName); - v.visitName(mQualname); - v.visitName(mModulename); - - v.visitTopo(mClosureType); - - v.visitHash(ShaHash(mOverloads.size())); - - for (auto o: mOverloads) { - o._visitCompilerVisibleInternals(v); - } - } - static Function* Make( std::string inName, std::string qualname, @@ -253,6 +232,28 @@ class Function : public Type { return concrete; } + template + void _visitCompilerVisibleInternals(const visitor_type& v) { + v.visitHash( + ShaHash(1, m_typeCategory) + + ShaHash(mIsNocompile ? 2 : 1) + + ShaHash(mIsEntrypoint ? 2 : 1) + ); + + v.visitName(m_name); + v.visitName(mRootName); + v.visitName(mQualname); + v.visitName(mModulename); + + v.visitTopo(mClosureType); + + v.visitHash(ShaHash(mOverloads.size())); + + for (auto o: mOverloads) { + o._visitCompilerVisibleInternals(v); + } + } + template void _visitContainedTypes(const visitor_type& visitor) { visitor(mClosureType); diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index e373edcbb..579738142 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1198,7 +1198,7 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { PyInstance::tp_iter : 0, // getiterfunc tp_iter; .tp_iternext = PyInstance::tp_iternext, // iternextfunc - .tp_methods = typeMethods(inType), // struct PyMethodDef* + .tp_methods = 0, // struct PyMethodDef* .tp_members = 0, // struct PyMemberDef* .tp_getset = 0, // struct PyGetSetDef* .tp_base = 0, // struct _typeobject* @@ -1250,16 +1250,8 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { ((PyObject*)&types[inType]->typeObj)->ob_type = incref(classMetaclass); } - PyType_Ready((PyTypeObject*)types[inType]); - - PyDict_SetItemString( - types[inType]->typeObj.tp_dict, - "__typed_python_category__", - categoryToPyString(inType->getTypeCategory()) - ); - if (!inType->isActivelyBeingDeserialized()) { - mirrorTypeInformationIntoPyType(inType, &types[inType]->typeObj); + finalizePyTypeObject(inType, (PyTypeObject*)types[inType]); } return (PyTypeObject*)types[inType]; @@ -1535,7 +1527,18 @@ bool PyInstance::typeCanBeSubclassed(Type* t) { } // static -void PyInstance::mirrorTypeInformationIntoPyType(Type* inType, PyTypeObject* pyType) { +void PyInstance::finalizePyTypeObject(Type* inType, PyTypeObject* pyType) { + // update the type methods + pyType->tp_methods = typeMethods(inType); + + PyType_Ready(pyType); + + PyDict_SetItemString( + pyType->tp_dict, + "__typed_python_category__", + categoryToPyString(inType->getTypeCategory()) + ); + specializeStatic(inType->getTypeCategory(), [&](auto* concrete_null_ptr) { typedef typename std::remove_reference::type py_instance_type; diff --git a/typed_python/PyInstance.hpp b/typed_python/PyInstance.hpp index dc7403d0d..f029f8ed3 100644 --- a/typed_python/PyInstance.hpp +++ b/typed_python/PyInstance.hpp @@ -641,7 +641,7 @@ class PyInstance { */ static PyTypeObject* typeObjInternal(Type* inType); - static void mirrorTypeInformationIntoPyType(Type* inType, PyTypeObject* pyType); + static void finalizePyTypeObject(Type* inType, PyTypeObject* pyType); static void mirrorTypeInformationIntoPyTypeConcrete(Type* inType, PyTypeObject* pyType); diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index bf769aff5..e58ede071 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1222,9 +1222,6 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur ixAndType.second->typeFinishedBeingDeserialized(); } - std::cout << "finalizing group:\n"; - std::cout << outGroup->repr() << "\n"; - outGroup->finalizeDeserializerGroup(); } @@ -1547,7 +1544,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( int isEntrypoint = 0; int isNocompile = 0; - std::vector classBases; + std::vector classBases; bool classIsFinal = false; std::vector > alternativeMembers; std::map classMethods, classStatics, classClassMethods, classPropertyFunctions; @@ -1631,7 +1628,7 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( t->name() + " of category " + t->getTypeCategoryString() ); } - classBases.push_back(((HeldClass*)t)->getClassType()); + classBases.push_back((HeldClass*)t); }); } else if (fieldNumber == 8) { assertWireTypesEqual(wireType, WireType::VARINT); @@ -1757,20 +1754,24 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( throw std::runtime_error("Shell is not a HeldClass"); } - if (types.size() != 1 || !types[0]->isClass()) { - throw std::runtime_error("Corrupt 'HeldClass' - needed a Class"); + if (types.size() != 1) { + throw std::runtime_error("Corrupt 'HeldClass' - needed a single type, not " + format(types.size())); + } + + if (!types[0]->isClass()) { + throw std::runtime_error("Corrupt 'HeldClass' - needed a Class, not " + types[0]->getTypeCategoryString()); } std::vector heldBases; for (auto b: classBases) { if (!b->isHeldClass()) { - throw std::runtime_error("Non-held base encountered!"); + throw std::runtime_error("Non-held base encountered: its " + b->getTypeCategoryString()); } heldBases.push_back((HeldClass*)b); } ((HeldClass*)blankShell)->initializeDuringDeserialization( - names[0], + "Held(" + names[0] + ")", heldBases, classIsFinal, classMembers, diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 52330d805..2368d1dbc 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -20,7 +20,7 @@ #include "MutuallyRecursiveTypeGroup.hpp" #include "ScopedIndenter.hpp" -#define TP_VERBOSE_SERIALIZE 1 +#define TP_VERBOSE_SERIALIZE 0 // virtual // serialize a python object. This function should work in any context. diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 96ce4c909..ac9a83728 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -679,7 +679,7 @@ void Type::typeFinishedBeingDeserialized() { // without it being finished. PyTypeObject* typeObj = PyInstance::typeObj(this); - PyInstance::mirrorTypeInformationIntoPyType(this, typeObj); + PyInstance::finalizePyTypeObject(this, typeObj); m_is_being_deserialized = false; } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 51772f311..92c616326 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -88,8 +88,6 @@ typedef uint8_t* instance_ptr; typedef void (*compiled_code_entrypoint)(instance_ptr, instance_ptr*); -void updateTypeRepForType(Type* t, PyTypeObject* pyType); - PyObject* getOrSetTypeResolver(PyObject* module = nullptr, PyObject* args = nullptr); enum class Maybe { @@ -946,7 +944,7 @@ class Type { TypeCategory m_typeCategory; - // this class is actively being deserialized. + // this class is actively being deserialized. bool m_is_being_deserialized; size_t m_size; diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 57545fe0c..7410a0793 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -3711,13 +3711,6 @@ static struct PyModuleDef moduledef = { .m_free = NULL }; -void updateTypeRepForType(Type* type, PyTypeObject* pyType) { - //deliberately leak the name. - pyType->tp_name = (new std::string(type->nameWithModule()))->c_str(); - - PyInstance::mirrorTypeInformationIntoPyType(type, pyType); -} - PyMODINIT_FUNC PyInit__types(void) { diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 55b166e56..d745ace17 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -45,7 +45,7 @@ Dict, Set, SerializationContext, EmbeddedMessage, serializeStream, deserializeStream, decodeSerializedObject, Forward, Final, Function, Entrypoint, TypeFunction, PointerTo, - SubclassOf, NotCompiled, resolveForwardDefinedType + SubclassOf, NotCompiled, resolveForwardDefinedType, identityHash ) from typed_python._types import ( @@ -2415,6 +2415,7 @@ def callF(x): else: return "FAILED" + # assert deserializeAndCall(aChildBytes) == "OK" assert callFunctionInFreshProcess(deserializeAndCall, (aChildBytes,)) == "OK" def test_call_method_dispatch_on_two_versions_of_self_referential_class_produced_differently(self): @@ -2499,6 +2500,30 @@ def test_identity_hash_of_lambda_doesnt_change_serialization(self): assert ser1 == ser2 + def test_deserialize_external_class_with_int_ref(self): + def deserializeAndCall(x): + class B(Class): + def f(self): + return x + + return B + + assert callFunctionInFreshProcess(deserializeAndCall, (1,)) is deserializeAndCall(1) + assert callFunctionInFreshProcess(deserializeAndCall, (1,)) is not deserializeAndCall(2) + assert callFunctionInFreshProcess(deserializeAndCall, (2,)) is deserializeAndCall(2) + + def test_deserialize_external_subclass(self): + class Base(Class): + pass + + def deserializeAndCall(): + class Child(Base): + pass + + return Child + + assert callFunctionInFreshProcess(deserializeAndCall, ()) is deserializeAndCall() + def test_call_method_dispatch_on_two_versions_of_same_class_with_recursion(self): Base = Forward(lambda: Base) @@ -2514,14 +2539,13 @@ def callF(x: Base): return x.f(10) def deserializeAndCall(): - # class Child(Base, Final): - # def f(self, x) -> int: - # return -1 + class Child(Base, Final): + def f(self, x) -> int: + return -1 - # assert isinstance(Child(), Base) + assert isinstance(Child(), Base) - return SerializationContext().serialize(Base) - #return SerializationContext().serialize(Child()) + return SerializationContext().serialize(Child()) childBytes = callFunctionInFreshProcess(deserializeAndCall, ()) From 616adaf42e5e6447feba36acc002b91629a531cf Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 20 Jun 2023 18:24:32 +0000 Subject: [PATCH 37/83] Failing on test_serialize_self_referencing_class_basic The problem is that when we have a set of recursive types that see themselves through function gobals, the function globals cannot be resolved to the concrete types until after identity hashing has happened. This is a problem because identity hashing requires that the object being hashed is stable! The solution is to modify the Type graph so that _everything_ is contained entirely within the forward type graph, with references to all other types (and python objects/classes) being modeled explicitly outside of the python interpreter. Then once the object is 'final' we can decide to merge it back into the version that's visible to the python interpreter. --- repro.py | 2 +- typed_python/FunctionOverload.hpp | 15 ++-- typed_python/MutuallyRecursiveTypeGroup.cpp | 9 +-- typed_python/PyInstance.cpp | 8 ++- typed_python/PyInstance.hpp | 4 +- ...onSerializationContext_deserialization.cpp | 71 +++++++++++-------- ...thonSerializationContext_serialization.cpp | 4 ++ typed_python/Type.cpp | 21 +++++- typed_python/Type.hpp | 3 +- typed_python/_types.cpp | 45 ++++++++---- typed_python/internals.py | 6 +- typed_python/type_construction_test.py | 11 ++- typed_python/types_serialization_test.py | 16 ++++- 13 files changed, 149 insertions(+), 66 deletions(-) diff --git a/repro.py b/repro.py index c378cd80a..76b9e8935 100644 --- a/repro.py +++ b/repro.py @@ -34,7 +34,7 @@ def reader(): def callF(x): return x.f(10) - assert callF(aChild) == 11 + assert callF(aChild) == -1 if sys.argv[1:] == ['r']: diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index b54174640..418547dd2 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -369,17 +369,20 @@ class FunctionOverload { } PyObject* o = PyCell_Get(nameAndGlobal.second); - Type* t = PyInstance::extractTypeFrom(o); + Type* t = o && PyType_Check(o) ? PyInstance::extractTypeFrom(o) : nullptr; if (t && t->isForwardDefined()) { - if (((Forward*)t)->isResolved()) { + if (t->isResolved()) { visitor.visitNamedTopo( nameAndGlobal.first, - ((Forward*)t)->forwardResolvesTo() + t->forwardResolvesTo() ); } else { // deliberately ignore non-resolved forwards + std::cout << "Deliberately ignoring " << nameAndGlobal.first << " -> " << TypeOrPyobj(t).name() << " since its not reoslved..\n"; } + } else if (t) { + visitor.visitNamedTopo(nameAndGlobal.first, t); } else { visitor.visitNamedTopo( nameAndGlobal.first, @@ -418,14 +421,16 @@ class FunctionOverload { Type* t = PyInstance::extractTypeFrom(val); if (t && t->isForwardDefined()) { - if (((Forward*)t)->isResolved()) { + if (t->isResolved()) { visitor.visitNamedTopo( name, - ((Forward*)t)->forwardResolvesTo() + t->forwardResolvesTo() ); } else { // deliberately ignore non-resolved forwards } + } else if (t) { + visitor.visitNamedTopo(name, t); } else { visitor.visitNamedTopo( name, diff --git a/typed_python/MutuallyRecursiveTypeGroup.cpp b/typed_python/MutuallyRecursiveTypeGroup.cpp index f0d9b5297..431fc2fad 100644 --- a/typed_python/MutuallyRecursiveTypeGroup.cpp +++ b/typed_python/MutuallyRecursiveTypeGroup.cpp @@ -102,7 +102,7 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { + canonical->repr() + "\n\nwhich has a different number of elements" + "\n\nmVisibilityType = " + visibilityTypeToStr(mVisibilityType) - + "\ncanonical->nmVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) + + "\ncanonical->mVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) ); } @@ -118,7 +118,7 @@ void MutuallyRecursiveTypeGroup::finalizeDeserializerGroup() { + canonical->repr() + "\n\nwhich has a different hash!" + "\n\nmVisibilityType = " + visibilityTypeToStr(mVisibilityType) - + "\ncanonical->nmVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) + + "\ncanonical->mVisibilityType = " + visibilityTypeToStr(canonical->mVisibilityType) ); } @@ -174,7 +174,7 @@ void MutuallyRecursiveTypeGroup::_computeHashAndInstall() { if (typeAndOrder.first.type() && mVisibilityType == VisibilityType::Identity) { Type* t = typeAndOrder.first.type(); t->setRecursiveTypeGroup( - this, + this, typeAndOrder.second, ShaHash(2) + this->hash() + ShaHash(typeAndOrder.second) ); @@ -257,7 +257,8 @@ std::string MutuallyRecursiveTypeGroup::repr(bool deep) { std::string levelPrefix(level, ' '); - s << levelPrefix << "group with hash " << group->hash().digestAsHexString() << ":\n"; + s << levelPrefix << "group with hash " << group->hash().digestAsHexString() << " and " + << "visibility=" << visibilityTypeToStr(mVisibilityType) << ":\n"; // sort lexically and then by level, so that // we can tell what's going on when we have a discrepancy diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 579738142..e604c1f5c 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1251,7 +1251,8 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { } if (!inType->isActivelyBeingDeserialized()) { - finalizePyTypeObject(inType, (PyTypeObject*)types[inType]); + finalizePyTypeObjectPhase1(inType, (PyTypeObject*)types[inType]); + finalizePyTypeObjectPhase2(inType, (PyTypeObject*)types[inType]); } return (PyTypeObject*)types[inType]; @@ -1527,12 +1528,14 @@ bool PyInstance::typeCanBeSubclassed(Type* t) { } // static -void PyInstance::finalizePyTypeObject(Type* inType, PyTypeObject* pyType) { +void PyInstance::finalizePyTypeObjectPhase1(Type* inType, PyTypeObject* pyType) { // update the type methods pyType->tp_methods = typeMethods(inType); PyType_Ready(pyType); +} +void PyInstance::finalizePyTypeObjectPhase2(Type* inType, PyTypeObject* pyType) { PyDict_SetItemString( pyType->tp_dict, "__typed_python_category__", @@ -1548,6 +1551,7 @@ void PyInstance::finalizePyTypeObject(Type* inType, PyTypeObject* pyType) { ); }); } + void PyInstance::mirrorTypeInformationIntoPyTypeConcrete(Type* inType, PyTypeObject* pyType) { //noop } diff --git a/typed_python/PyInstance.hpp b/typed_python/PyInstance.hpp index f029f8ed3..76190c617 100644 --- a/typed_python/PyInstance.hpp +++ b/typed_python/PyInstance.hpp @@ -641,7 +641,9 @@ class PyInstance { */ static PyTypeObject* typeObjInternal(Type* inType); - static void finalizePyTypeObject(Type* inType, PyTypeObject* pyType); + static void finalizePyTypeObjectPhase1(Type* inType, PyTypeObject* pyType); + + static void finalizePyTypeObjectPhase2(Type* inType, PyTypeObject* pyType); static void mirrorTypeInformationIntoPyTypeConcrete(Type* inType, PyTypeObject* pyType); diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index e58ede071..3fb4e2df3 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1046,6 +1046,46 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur }); } else if (fieldNumber == 4) { + // now that we've constructed the type diagram, walk over all the types + // and finalize them. We have to do this before we wire function globals + // because sometime the function globals hold python references to type objects + // and the type objects have to be filled out completely and have PyType_Ready + // called on them before they are valid GC objects. + if (actuallyBuildGroup) { + // recompute type names + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->recomputeName(); + } + + // do post-initialization + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto ixAndType: indicesOfNativeTypes) { + if (ixAndType.second->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > indicesOfNativeTypes.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } + } + + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->finalizeType(); + } + + for (auto ixAndType: indicesOfNativeTypes) { + ixAndType.second->typeFinishedBeingDeserializedPhase1(); + } + } + // fill out the globals of function objects. we have to deserialize these // in a separate final pass because they can have references to existing // native types, and we need those objects to be filled out fully. @@ -1189,37 +1229,8 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } if (actuallyBuildGroup) { - // recompute type names - for (auto ixAndType: indicesOfNativeTypes) { - ixAndType.second->recomputeName(); - } - - // do post-initialization - bool anyUpdated = true; - size_t passCt = 0; - while (anyUpdated) { - anyUpdated = false; - for (auto ixAndType: indicesOfNativeTypes) { - if (ixAndType.second->postInitialize()) { - anyUpdated = true; - } - } - passCt += 1; - - // we can run this algorithm until all type sizes have stabilized. Conceivably we - // could introduce an error that would cause this to not converge - this should - // detect that. - if (passCt > indicesOfNativeTypes.size() * 2 + 10) { - throw std::runtime_error("Type size graph is not stabilizing."); - } - } - - for (auto ixAndType: indicesOfNativeTypes) { - ixAndType.second->finalizeType(); - } - for (auto ixAndType: indicesOfNativeTypes) { - ixAndType.second->typeFinishedBeingDeserialized(); + ixAndType.second->typeFinishedBeingDeserializedPhase2(); } outGroup->finalizeDeserializerGroup(); diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 2368d1dbc..365a35074 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -405,6 +405,10 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec throw std::runtime_error("Can't serialize a mutually recursive type group with an invalid hash."); } + if (group->visibilityType() != VisibilityType::Identity) { + throw std::runtime_error("shouldn't be serializing a compiler identity group."); + } + b.writeBeginCompound(fieldNumber); // group memo uint32_t id; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index ac9a83728..cd131eae9 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -508,6 +508,11 @@ void Type::attemptToResolve() { typeAndSource.first->finalizeType(); } + std::cout << "resolving group:\n"; + for (auto typeAndSource: resolutionSource) { + std::cout << " " << TypeOrPyobj(typeAndSource.first).name() << " from " << TypeOrPyobj(typeAndSource.second).name() << "\n"; + } + // now internalize the types by their hash. For each type, we compute a hash // and then look to see if we've seen it before. We build a lookup table from // each existing type to the internalized type, and then do the same process we did @@ -671,7 +676,19 @@ void Type::attemptAutoresolveWrite() { } } -void Type::typeFinishedBeingDeserialized() { +void Type::typeFinishedBeingDeserializedPhase1() { + if (m_is_being_deserialized) { + // during deserialization we have to defer anything that actually + // looks hard at the type and copies data into the PyTypeObject + // because its not ready yet. This allows us to refer to the object + // without it being finished. + PyTypeObject* typeObj = PyInstance::typeObj(this); + + PyInstance::finalizePyTypeObjectPhase1(this, typeObj); + } +} + +void Type::typeFinishedBeingDeserializedPhase2() { if (m_is_being_deserialized) { // during deserialization we have to defer anything that actually // looks hard at the type and copies data into the PyTypeObject @@ -679,7 +696,7 @@ void Type::typeFinishedBeingDeserialized() { // without it being finished. PyTypeObject* typeObj = PyInstance::typeObj(this); - PyInstance::finalizePyTypeObject(this, typeObj); + PyInstance::finalizePyTypeObjectPhase2(this, typeObj); m_is_being_deserialized = false; } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 92c616326..63e12ccec 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -924,7 +924,8 @@ class Type { return m_is_being_deserialized; } - void typeFinishedBeingDeserialized(); + void typeFinishedBeingDeserializedPhase1(); + void typeFinishedBeingDeserializedPhase2(); protected: Type(TypeCategory in_typeCategory) : diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 7410a0793..15ab829e7 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -938,22 +938,41 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { Function* resType; if (PyTuple_Size(args) == 2) { - PyObjectHolder a0(PyTuple_GetItem(args, 0)); - PyObjectHolder a1(PyTuple_GetItem(args, 1)); + if (PyUnicode_Check(PyTuple_GetItem(args, 1))) { + PyObjectHolder a0(PyTuple_GetItem(args, 0)); + Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); - Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); - Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); + if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); + throw PythonExceptionSet(); + } - if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); - throw PythonExceptionSet(); - } - if (!t1 || t1->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_SetString(PyExc_TypeError, "Expected second argument to be a function"); - throw PythonExceptionSet(); - } + if (PyUnicode_AsUTF8(PyTuple_GetItem(args, 1)) == std::string("Entrypoint")) { + resType = ((Function*)t0)->withEntrypoint(true); + } else if (PyUnicode_AsUTF8(PyTuple_GetItem(args, 1)) == std::string("Nocompile")) { + resType = ((Function*)t0)->withEntrypoint(true); + } else { + PyErr_SetString(PyExc_TypeError, "Expected string argument to be Entrypoint or Nocompile"); + throw PythonExceptionSet(); + } + } else { + PyObjectHolder a0(PyTuple_GetItem(args, 0)); + PyObjectHolder a1(PyTuple_GetItem(args, 1)); + + Type* t0 = PyInstance::unwrapTypeArgToTypePtr(a0); + Type* t1 = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t0 || t0->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected first argument to be a function"); + throw PythonExceptionSet(); + } + if (!t1 || t1->getTypeCategory() != Type::TypeCategory::catFunction) { + PyErr_SetString(PyExc_TypeError, "Expected second argument to be a function"); + throw PythonExceptionSet(); + } - resType = Function::merge((Function*)t0, (Function*)t1); + resType = Function::merge((Function*)t0, (Function*)t1); + } } else { PyObjectHolder nameObj(PyTuple_GetItem(args, 0)); if (!PyUnicode_Check(nameObj)) { diff --git a/typed_python/internals.py b/typed_python/internals.py index 47b5bad7e..6c23059ea 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -170,11 +170,11 @@ def makeFunctionType( if f.isEntrypoint: # reapply the entrypoint flag - res = type(res().withEntrypoint(True)) + res = typed_python._types.Function(res, 'Entrypoint') if f.isNocompile: - # reapply the entrypoint flag - res = type(res().withNocompile(True)) + # reapply the notcompiled flag + res = typed_python._types.Function(res, 'Nocompile') return res diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index dd2c1bdaa..755175ed3 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,7 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held + Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled ) from typed_python.test_util import CodeEvaluator @@ -99,6 +99,15 @@ def callIt(x: B): assert not isForwardDefined(type(callIt)) +def test_class_with_entrypointed_recursive_method(): + class C(Class): + @NotCompiled + def f(self): + return C().g() + + c = C() + + def test_autoresolve_forwards_with_nonforwards(): class C(Class): def f(self): diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index d745ace17..205e13040 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2013,7 +2013,7 @@ def __add__(self, other: Base) -> Base: assert isinstance(Base() + Base() + Base(), Child) def test_serialize_self_referencing_class_basic(self): - def g(x): + def g2(x): return 10 @TypeFunction @@ -2025,10 +2025,20 @@ def s(self): @Entrypoint def g(self): - return g(10) + return g2(10) return C_ + print(recursiveTypeGroupRepr(C(int), "identity")) + print(recursiveTypeGroupRepr(C(int), "compiler")) + from typed_python._types import checkForHashInstability + checkForHashInstability() + #from typed_python._types import typeWalkRecord + #print(typeWalkRecord(type(C(int).s), 'identity')) + #print(typeWalkRecord(type(C(int).s), 'compiler')) + + return + c = callFunctionInFreshProcess(C(int), ()) assert c.g() == 10 @@ -2267,7 +2277,7 @@ def f(self) -> Cls: callFunctionInFreshProcess(getCls, ()) - def test_can_deserialize_forward_class_methods_tp_class(self): + def test_can_deserialize_forward_class_methods_tp_class_2(self): Cls = Forward("Cls") @Cls.define From da08d891317473e5dec505ba274a9660f6d09563 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 27 Jun 2023 17:03:03 +0000 Subject: [PATCH 38/83] Start to work through FunctionGlobal and CompilerVisiblePyObj model --- typed_python/CompilerVisiblePyObj.hpp | 99 ++++++++++++++++ typed_python/FunctionGlobal.hpp | 155 ++++++++++++++++++++++++++ typed_python/FunctionOverload.hpp | 60 ++++------ typed_python/FunctionType.hpp | 2 +- 4 files changed, 274 insertions(+), 42 deletions(-) create mode 100644 typed_python/CompilerVisiblePyObj.hpp create mode 100644 typed_python/FunctionGlobal.hpp diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp new file mode 100644 index 000000000..78dcd5353 --- /dev/null +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -0,0 +1,99 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "Instance.hpp" + + +/********************************* +CompilerVisiblePyObject + +a representation of a python object that's owned by TypedPython. We hold these by +pointer and leak them indiscriminately - like Type objects they're considered to be permanent +and singletonish + +**********************************/ + +class CompilerVisiblePyObj { + enum class Kind { + Uninitialized = 0, + Type = 1, + Instance = 2, + PyTuple = 3, + }; + + CompilerVisiblePyObj() : + mKind(Kind::Uninitialized), + mType(nullptr) + { + } + +public: + static CompilerVisiblePyObj* Type(Type* t) { + CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + + res->mKind = Kind::Type; + res->mType = t; + + return res; + } + + static CompilerVisiblePyObj* Instance(Instance i) { + CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + + res->mKind = Kind::Instance; + res->mInstance = i; + + return res; + } + + static CompilerVisiblePyObj* PyTuple() { + CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + + res->mKind = Kind::PyTuple; + + return res; + } + + void append(CompilerVisiblePyObj* elt) { + if (mKind != Kind::PyTuple) { + throw std::runtime_error("Expected a PyTuple"); + } + + mElements.push_back(elt); + } + + const std::vector& elements() const { + return mElements; + } + + Type* getType() const { + return mType; + } + + const Instance& getInstance() const { + return mInstance; + } + +private: + Kind mKind; + + Type* mType; + Instance mInstance; + + std::vector mElements; +}; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp new file mode 100644 index 000000000..b926e1586 --- /dev/null +++ b/typed_python/FunctionGlobal.hpp @@ -0,0 +1,155 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "CompilerVisiblePyObj.hpp" + + +class FunctionGlobal { + enum class GlobalType { + Unbound = 1, // this global is not bound to anything - its a Name error + NamedModuleMember = 2, // this global is a member of a particular module + Constant = 3, // this global is bound to a constant that we resolved to at some point + ForwardInDict = 4, // this global is an unresolved variable residing in some python dict + ForwardInCell = 5, // this global is an unresolved variable residing in some PyCell + }; + + FunctionGlobal() : + mKind(GlobalType::Unbound), + mModuleDictOrCell(nullptr) + { + } + + FunctionGlobal( + GlobalType inKind, + PyObject* dictOrCell, + std::string name, + CompilerVisiblePyObj* constant + ) : + mKind(inKind), + mModuleDictOrCell(incref(dictOrCell)), + mName(name), + mConstant(constant) + { + } + +public: + static FunctionGlobal Unbound() { + return FunctionGlobal(); + } + + static FunctionGlobal Constant(CompilerVisiblePyObj* constant) { + return FunctionGlobal( + GlobalType::Constant, + nullptr, + "", + constant + ); + } + + static FunctionGlobal NamedModuleMember(PyObject* moduleDict, std::string name) { + if (!PyDict_Check(moduleDict)) { + throw std::runtime_error("NamedModuleMember requires a Python dict object"); + } + + return FunctionGlobal( + GlobalType::NamedModuleMember, + moduleDict, + name, + nullptr + ); + } + + static FunctionGlobal Constant(CompilerVisiblePyObj* constant) { + return FunctionGlobal( + GlobalType::Constant, + nullptr, + "", + constant + ); + } + + static FunctionGlobal ForwardInDict(PyObject* dict, std::string name) { + if (!PyDict_Check(dict)) { + throw std::runtime_error("ForwardInDict requires a Python dict object"); + } + + return FunctionGlobal( + GlobalType::ForwardInDict, + dict, + name, + nullptr + ); + } + + static FunctionGlobal ForwardInCell(PyObject* cell, std::string name) { + if (!PyCell_Check(cell)) { + throw std::runtime_error("ForwardInCell requires a Python cell object"); + } + + return FunctionGlobal( + GlobalType::ForwardInCell, + cell, + name, + nullptr + ); + } + + bool isUnbound() const { + return mKind == GlobalType::Unbound; + } + + bool isNamedModuleMember() const { + return mKind == GlobalType::NamedModuleMember; + } + + bool isConstant() const { + return mKind == GlobalType::Constant; + } + + bool isForwardInDict() const { + return mKind == GlobalType::ForwardInDict; + } + + bool isForwardInCell() const { + return mKind == GlobalType::ForwardInCell; + } + + GlobalType getKind() const { + return mKind; + } + + CompilerVisiblePyObj* getConstant() const { + return mConstant; + } + + const std::string& getName() const { + return mName; + } + + PyObject* getModuleDictOrCell() const { + return mModuleDictOrCell; + } + +private: + GlobalType mKind; + + CompilerVisiblePyObj* mConstant; + + std::string mName; + PyObject* mModuleDictOrCell; +}; diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 418547dd2..5ba7a56b4 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2022 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,16 +25,16 @@ #include "ClosureVariableBinding.hpp" #include "FunctionArg.hpp" #include "CompiledSpecialization.hpp" +#include "FunctionGlobals.hpp" class FunctionOverload { public: FunctionOverload( PyObject* pyFuncCode, - PyObject* pyFuncGlobals, PyObject* pyFuncDefaults, PyObject* pyFuncAnnotations, - const std::map& pyFuncGlobalsInCells, + const std::map& inGlobals, const std::vector& pyFuncClosureVarnames, const std::map closureBindings, Type* returnType, @@ -43,10 +43,9 @@ class FunctionOverload { Type* methodOf ) : mFunctionCode(pyFuncCode), - mFunctionGlobals(pyFuncGlobals), mFunctionDefaults(pyFuncDefaults), mFunctionAnnotations(pyFuncAnnotations), - mFunctionGlobalsInCells(pyFuncGlobalsInCells), + mGlobals(inGlobals), mFunctionClosureVarnames(pyFuncClosureVarnames), mReturnType(returnType), mSignatureFunction(pySignatureFunction), @@ -88,7 +87,7 @@ class FunctionOverload { other.increfAllPyObjects(); mFunctionCode = other.mFunctionCode; - mFunctionGlobals = other.mFunctionGlobals; + mGlobals = other.mGlobals; mFunctionDefaults = other.mFunctionDefaults; mFunctionAnnotations = other.mFunctionAnnotations; mSignatureFunction = other.mSignatureFunction; @@ -123,10 +122,9 @@ class FunctionOverload { return FunctionOverload( mFunctionCode, - mFunctionGlobals, mFunctionDefaults, mFunctionAnnotations, - mFunctionGlobalsInCells, + mGlobals, mFunctionClosureVarnames, bindings, mReturnType, @@ -139,10 +137,9 @@ class FunctionOverload { FunctionOverload withMethodOf(Type* methodOf) const { return FunctionOverload( mFunctionCode, - mFunctionGlobals, mFunctionDefaults, mFunctionAnnotations, - mFunctionGlobalsInCells, + mGlobals, mFunctionClosureVarnames, mClosureBindings, mReturnType, @@ -155,10 +152,9 @@ class FunctionOverload { FunctionOverload withClosureBindings(const std::map &bindings) const { return FunctionOverload( mFunctionCode, - mFunctionGlobals, mFunctionDefaults, mFunctionAnnotations, - mFunctionGlobalsInCells, + mGlobals, mFunctionClosureVarnames, bindings, mReturnType, @@ -359,9 +355,9 @@ class FunctionOverload { visitor.visitHash(ShaHash(2)); } - visitor.visitHash(ShaHash(mFunctionGlobalsInCells.size())); + visitor.visitHash(ShaHash(mGlobals.size())); - for (auto nameAndGlobal: mFunctionGlobalsInCells) { + for (auto nameAndGlobal: mGlobals) { if (!PyCell_Check(nameAndGlobal.second)) { throw std::runtime_error( "A global in mFunctionGlobalsInCells is somehow not a cell" @@ -541,11 +537,8 @@ class FunctionOverload { if (mFunctionCode < other.mFunctionCode) { return true; } if (mFunctionCode > other.mFunctionCode) { return false; } - if (mFunctionGlobals < other.mFunctionGlobals) { return true; } - if (mFunctionGlobals > other.mFunctionGlobals) { return false; } - - if (mFunctionGlobalsInCells < other.mFunctionGlobalsInCells) { return true; } - if (mFunctionGlobalsInCells > other.mFunctionGlobalsInCells) { return false; } + if (mGlobals < other.mGlobals) { return true; } + if (mGlobals > other.mGlobals) { return false; } if (mClosureBindings < other.mClosureBindings) { return true; } if (mClosureBindings > other.mClosureBindings) { return false; } @@ -590,8 +583,8 @@ class FunctionOverload { return mFunctionGlobals; } - const std::map& getFunctionGlobalsInCells() const { - return mFunctionGlobalsInCells; + const std::map& getGlobals() const { + return mGlobals; } /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. @@ -1080,10 +1073,9 @@ class FunctionOverload { return FunctionOverload( functionCode, - functionGlobals, functionDefaults, functionAnnotations, - functionGlobalsInCellsRaw, + functionGlobals, closureVarnames, closureBindings, returnType, @@ -1099,10 +1091,6 @@ class FunctionOverload { incref(mFunctionDefaults); incref(mFunctionAnnotations); incref(mSignatureFunction); - - for (auto nameAndOther: mFunctionGlobalsInCells) { - incref(nameAndOther.second); - } } void decrefAllPyObjects() { @@ -1111,23 +1099,13 @@ class FunctionOverload { decref(mFunctionDefaults); decref(mFunctionAnnotations); decref(mSignatureFunction); - - for (auto nameAndOther: mFunctionGlobalsInCells) { - decref(nameAndOther.second); - } } private: - PyObject* mFunctionCode; - - PyObject* mFunctionGlobals; + // every global we access not through our closure + std::map mGlobals; - // globals that are stored in cells. This happens when class objects - // are defined inside of function scopes. We assume that anything in their - // closure is global (and therefore constant) but it may not be defined yet, - // so we can't just pull the value out and stick it in the function closure - // itself. Each value in the map is guaranteed to be a 'cell' object. - std::map mFunctionGlobalsInCells; + PyObject* mFunctionCode; // the order (by name) of the variables in the __closure__ of the original // function. This is the order that the python code will expect. @@ -1147,7 +1125,7 @@ class FunctionOverload { Type* mReturnType; - // if we are a method of a class, what class? Used to + // if we are a method of a class, what class? Type* mMethodOf; std::vector mArgs; diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 559b9f0d8..06fab0b2c 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2022 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From a293dbf9dc9ace4947d71a553c48aefccc80c40e Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 29 Jun 2023 17:48:36 +0000 Subject: [PATCH 39/83] New FunctionGlobal stuff compiles even if it doesn't work yet. --- typed_python/CompilerVisiblePyObj.hpp | 31 +- typed_python/FunctionGlobal.hpp | 298 +++++++++++++++++- typed_python/FunctionOverload.cpp | 127 +++++--- typed_python/FunctionOverload.hpp | 263 +++------------- typed_python/FunctionType.hpp | 4 + .../ModuleRepresentationCopyContext.hpp | 14 +- typed_python/PyFunctionInstance.cpp | 65 ++-- typed_python/PyFunctionInstance.hpp | 14 +- ...onSerializationContext_deserialization.cpp | 25 -- ...thonSerializationContext_serialization.cpp | 24 -- typed_python/_types.cpp | 13 +- typed_python/internals.py | 39 +-- 12 files changed, 476 insertions(+), 441 deletions(-) diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 78dcd5353..d7eaad899 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -16,6 +16,8 @@ #pragma once +#include +#include "Type.hpp" #include "Instance.hpp" @@ -81,19 +83,40 @@ class CompilerVisiblePyObj { return mElements; } - Type* getType() const { + ::Type* getType() const { return mType; } - const Instance& getInstance() const { + const ::Instance& getInstance() const { return mInstance; } + template + void _visitReferencedTypes(const visitor_type& v) { + if (mKind == Kind::Type) { + v(mType); + return; + } + + if (mKind == Kind::Instance) { + // TODO: what to do here? + } + + if (mKind == Kind::PyTuple) { + // TODO: what to do here? + } + } + + template + void _visitCompilerVisibleInternals(const visitor_type& visitor) { + + } + private: Kind mKind; - Type* mType; - Instance mInstance; + ::Type* mType; + ::Instance mInstance; std::vector mElements; }; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index b926e1586..d419bcdc9 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -16,6 +16,7 @@ #pragma once + #include "CompilerVisiblePyObj.hpp" @@ -28,12 +29,6 @@ class FunctionGlobal { ForwardInCell = 5, // this global is an unresolved variable residing in some PyCell }; - FunctionGlobal() : - mKind(GlobalType::Unbound), - mModuleDictOrCell(nullptr) - { - } - FunctionGlobal( GlobalType inKind, PyObject* dictOrCell, @@ -48,6 +43,12 @@ class FunctionGlobal { } public: + FunctionGlobal() : + mKind(GlobalType::Unbound), + mModuleDictOrCell(nullptr) + { + } + static FunctionGlobal Unbound() { return FunctionGlobal(); } @@ -61,6 +62,52 @@ class FunctionGlobal { ); } + // return an approriate FunctionGlobal given that we are indexing into 'funcGlobals' + // with a dot-variable sequence. Returns a pair of (dotSequence, FunctionGlobal) that + // we actually want to use + static std::pair DottedGlobalsLookup( + PyObject* funcGlobals, + std::string dotSequence + ) { + if (!PyDict_Check(funcGlobals)) { + throw std::runtime_error("Can't handle a non-dict function globals"); + } + + std::string shortGlobalName = dotSequence; + + size_t indexOfDot = shortGlobalName.find('.'); + if (indexOfDot != std::string::npos) { + shortGlobalName = shortGlobalName.substr(0, indexOfDot); + } + + PyObject* globalVal = PyDict_GetItemString(funcGlobals, shortGlobalName.c_str()); + + if (!globalVal) { + PyObject* builtins = PyDict_GetItemString(funcGlobals, "__builtins__"); + if (builtins && PyDict_Check(builtins) && PyDict_GetItemString(builtins, shortGlobalName.c_str())) { + return std::make_pair( + shortGlobalName, + FunctionGlobal::NamedModuleMember(builtins, shortGlobalName) + ); + } + } + + // TODO: this should be more precise - we need to decide what to do if + // this is a named global in a globally visible module vs a module + // still being defined, or whatever + return std::make_pair( + shortGlobalName, + FunctionGlobal::ForwardInDict(funcGlobals, shortGlobalName) + ); + } + + // construct a Global from a cell. If the cell is defined, we can resolve immediately + static FunctionGlobal FromCell(PyObject* cell) { + // pass + throw std::runtime_error("FunctionGlobal::FromCell not implemented yet"); + } + + static FunctionGlobal NamedModuleMember(PyObject* moduleDict, std::string name) { if (!PyDict_Check(moduleDict)) { throw std::runtime_error("NamedModuleMember requires a Python dict object"); @@ -74,15 +121,6 @@ class FunctionGlobal { ); } - static FunctionGlobal Constant(CompilerVisiblePyObj* constant) { - return FunctionGlobal( - GlobalType::Constant, - nullptr, - "", - constant - ); - } - static FunctionGlobal ForwardInDict(PyObject* dict, std::string name) { if (!PyDict_Check(dict)) { throw std::runtime_error("ForwardInDict requires a Python dict object"); @@ -129,6 +167,10 @@ class FunctionGlobal { return mKind == GlobalType::ForwardInCell; } + PyObject* getValueAsPyobj() { + throw std::runtime_error("FunctionGlobal::getValueAsPyobj not implemented"); + } + GlobalType getKind() const { return mKind; } @@ -145,6 +187,232 @@ class FunctionGlobal { return mModuleDictOrCell; } + template + void _visitReferencedTypes(const visitor_type& visitor) { + if (isUnbound()) { + return; + } + + if (isConstant()) { + mConstant->_visitReferencedTypes(visitor); + return; + } + + //TODO: what should we do here? + return; + } + + template + void _visitCompilerVisibleInternals(const visitor_type& visitor) { + if (isUnbound()) { + return; + } + + if (isNamedModuleMember()) { + // TODO: need to know whether we're an identity or compiler visitor? + return; + } + + if (isConstant()) { + mConstant->_visitCompilerVisibleInternals(visitor); + return; + } + + if (isForwardInDict()) { + return; + } + + if (isForwardInCell()) { + return; + } + + // if (!PyCell_Check(nameAndGlobal.second)) { + // throw std::runtime_error( + // "A global in mFunctionGlobalsInCells is somehow not a cell" + // ); + // } + + // PyObject* o = PyCell_Get(nameAndGlobal.second); + // Type* t = o && PyType_Check(o) ? PyInstance::extractTypeFrom(o) : nullptr; + + // if (t && t->isForwardDefined()) { + // if (t->isResolved()) { + // visitor.visitNamedTopo( + // nameAndGlobal.first, + // t->forwardResolvesTo() + // ); + // } else { + // // deliberately ignore non-resolved forwards + // std::cout << "Deliberately ignoring " << nameAndGlobal.first << " -> " << TypeOrPyobj(t).name() << " since its not reoslved..\n"; + // } + // } else if (t) { + // visitor.visitNamedTopo(nameAndGlobal.first, t); + // } else { + // visitor.visitNamedTopo( + // nameAndGlobal.first, + // nameAndGlobal.second + // ); + // } + + + + // or alternatively + // + // _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { + // Type* t = PyInstance::extractTypeFrom(val); + + // if (t && t->isForwardDefined()) { + // if (t->isResolved()) { + // visitor.visitNamedTopo( + // name, + // t->forwardResolvesTo() + // ); + // } else { + // // deliberately ignore non-resolved forwards + // } + // } else if (t) { + // visitor.visitNamedTopo(name, t); + // } else { + // visitor.visitNamedTopo( + // name, + // val + // ); + // } + // }); + + } + + bool isUnresolved(bool insistForwardsResolved) { + throw std::runtime_error("FunctionGlobal::isUnresolved not implemented yet"); + + // auto it = mFunctionGlobalsInCells.find(name); + // if (it != mFunctionGlobalsInCells.end()) { + // if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + // if (insistForwardsResolved) { + // PyObject* o = PyCell_Get(it->second); + // Type* t = PyInstance::extractTypeFrom(o); + // if (t && t->isForwardDefined()) { + // return true; + // } + // } + + // return false; + // } + + // return true; + // } + + // if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { + // if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { + // if (insistForwardsResolved) { + // PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + // Type* t = PyInstance::extractTypeFrom(o); + // if (t && t->isForwardDefined()) { + // return true; + // } + // } + + // return false; + // } + + // PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); + // if (builtins && PyDict_GetItemString(builtins, name.c_str())) { + // return false; + // } + // } + + // return true; + } + + void autoresolveGlobal( + const std::set& resolvedForwards + ) { + throw std::runtime_error("FunctionGlobal::autoresolveGlobal not implemented yet"); + + // if (it != mFunctionGlobalsInCells.end()) { + // if (PyCell_Check(it->second) && PyCell_GET(it->second)) { + // if (PyType_Check(PyCell_GET(it->second))) { + // Type* cellContents = PyInstance::extractTypeFrom( + // (PyTypeObject*)PyCell_Get(it->second) + // ); + + // if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + // PyCell_Set( + // it->second, + // (PyObject*)PyInstance::typeObj( + // cellContents->forwardResolvesTo() + // ) + // ); + // } + // } + // } + + // return; + // } + + // PyObject* dictVal = PyDict_GetItemString(mFunctionGlobals, name.c_str()); + // if (dictVal && PyType_Check(dictVal)) { + // Type* cellContents = PyInstance::extractTypeFrom( + // (PyTypeObject*)dictVal + // ); + + // if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + // PyDict_SetItemString( + // mFunctionGlobals, + // name.c_str(), + // (PyObject*)PyInstance::typeObj( + // cellContents->forwardResolvesTo() + // ) + // ); + // } + // } + } + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + // context.serializePythonObject(nameAndCell.second, buffer, varIx++); + throw std::runtime_error("FunctionGlobal::serialize not implemented yet"); + } + + template + static FunctionGlobal deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + throw std::runtime_error("FunctionGlobal::deserialize not implemented yet"); + // functionGlobalsInCells[last].steal(context.deserializePythonObject(buffer, wireType)); + // functionGlobalsInCellsRaw[last] = functionGlobalsInCells[last]; + } + + bool operator<(const FunctionGlobal& g) const { + if (mKind < g.mKind) { + return true; + } + if (mKind > g.mKind) { + return false; + } + + if (mConstant < g.mConstant) { + return true; + } + if (mConstant > g.mConstant) { + return false; + } + + if (mName < g.mName) { + return true; + } + if (mName > g.mName) { + return false; + } + + if (mModuleDictOrCell < g.mModuleDictOrCell) { + return true; + } + if (mModuleDictOrCell > g.mModuleDictOrCell) { + return false; + } + + return false; + } + private: GlobalType mKind; diff --git a/typed_python/FunctionOverload.cpp b/typed_python/FunctionOverload.cpp index 4151becb6..e30cd6f06 100644 --- a/typed_python/FunctionOverload.cpp +++ b/typed_python/FunctionOverload.cpp @@ -2,12 +2,33 @@ /* static */ -PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr closureData) const { +PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr closureData) { if (mCachedFunctionObj) { return incref(mCachedFunctionObj); } - PyObject* res = PyFunction_New(mFunctionCode, mFunctionGlobals); + std::set globalsInCells( + mFunctionClosureVarnames.begin(), + mFunctionClosureVarnames.end() + ); + + PyObjectStealer funcGlobals(PyDict_New()); + + for (auto& nameAndGlobal: mGlobals) { + if (globalsInCells.find(nameAndGlobal.first) == globalsInCells.end()) { + PyObject* val = nameAndGlobal.second.getValueAsPyobj(); + + if (val) { + PyDict_SetItemString( + (PyObject*)funcGlobals, + nameAndGlobal.first.c_str(), + val + ); + } + } + } + + PyObject* res = PyFunction_New(mFunctionCode, (PyObject*)funcGlobals); if (!res) { throw PythonExceptionSet(); @@ -42,64 +63,68 @@ PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr clo for (long k = 0; k < closureVarCount; k++) { std::string varname = mFunctionClosureVarnames[k]; - auto globalIt = mFunctionGlobalsInCells.find(varname); - if (globalIt != mFunctionGlobalsInCells.end()) { - if (varname == "__class__" && getMethodOf()) { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - PyCell_New((PyObject*)PyInstance::typeObj( - ((HeldClass*)getMethodOf())->getClassType() - )) - ); - } else { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - incref(globalIt->second) - ); - } + if (varname == "__class__" && getMethodOf()) { + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New((PyObject*)PyInstance::typeObj( + ((HeldClass*)getMethodOf())->getClassType() + )) + ); } else { - auto bindingIt = mClosureBindings.find(varname); - if (bindingIt == mClosureBindings.end()) { - throw std::runtime_error("Closure variable " + varname + " not in globals or closure bindings."); - } + auto globalIt = mGlobals.find(varname); + if (globalIt != mGlobals.end()) { + PyObject* globalVal = globalIt->second.getValueAsPyobj(); + + if (globalVal) { + PyTuple_SetItem( + (PyObject*)closureTup, + k, + incref(globalVal) + ); + } + } else { + auto bindingIt = mClosureBindings.find(varname); + if (bindingIt == mClosureBindings.end()) { + throw std::runtime_error("Closure variable " + varname + " not in globals or closure bindings."); + } - ClosureVariableBinding binding = bindingIt->second; + ClosureVariableBinding binding = bindingIt->second; - Instance bindingValue = binding.extractValueOrContainingClosure(closureType, closureData); + Instance bindingValue = binding.extractValueOrContainingClosure(closureType, closureData); - if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catPyCell) { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - incref( - ((PyCellType*)bindingValue.type())->getPyObj(bindingValue.data()) - ) - ); - } else { - // it's not a cell. We have to encode it as a PyCell for the interpreter - // to be able to handle it. - PyObjectHolder newCellContents; - - if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catTypedCell) { - newCellContents.steal( - PyInstance::extractPythonObject( - ((TypedCellType*)bindingValue.type())->get(bindingValue.data()), - ((TypedCellType*)bindingValue.type())->getHeldType() + if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catPyCell) { + PyTuple_SetItem( + (PyObject*)closureTup, + k, + incref( + ((PyCellType*)bindingValue.type())->getPyObj(bindingValue.data()) ) ); } else { - newCellContents.steal( - PyInstance::fromInstance(bindingValue) + // it's not a cell. We have to encode it as a PyCell for the interpreter + // to be able to handle it. + PyObjectHolder newCellContents; + + if (bindingValue.type()->getTypeCategory() == Type::TypeCategory::catTypedCell) { + newCellContents.steal( + PyInstance::extractPythonObject( + ((TypedCellType*)bindingValue.type())->get(bindingValue.data()), + ((TypedCellType*)bindingValue.type())->getHeldType() + ) + ); + } else { + newCellContents.steal( + PyInstance::fromInstance(bindingValue) + ); + } + + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New(newCellContents) ); } - - PyTuple_SetItem( - (PyObject*)closureTup, - k, - PyCell_New(newCellContents) - ); } } } diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 5ba7a56b4..8db7dd001 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -25,7 +25,7 @@ #include "ClosureVariableBinding.hpp" #include "FunctionArg.hpp" #include "CompiledSpecialization.hpp" -#include "FunctionGlobals.hpp" +#include "FunctionGlobal.hpp" class FunctionOverload { @@ -105,8 +105,6 @@ class FunctionOverload { mMinPositionalArgs = other.mMinPositionalArgs; mMaxPositionalArgs = other.mMaxPositionalArgs; - mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; - mCachedFunctionObj = other.mCachedFunctionObj; } @@ -285,31 +283,9 @@ class FunctionOverload { visitor(mMethodOf); } - for (auto nameAndGlobal: mFunctionGlobalsInCells) { - if (!PyCell_Check(nameAndGlobal.second)) { - throw std::runtime_error( - "A global in mFunctionGlobalsInCells is somehow not a cell" - ); - } - - PyObject* cellContents = PyCell_Get(nameAndGlobal.second); - - if (cellContents && PyType_Check(cellContents)) { - Type* t = PyInstance::extractTypeFrom(cellContents); - if (t) { - visitor(t); - } - } + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second._visitReferencedTypes(visitor); } - - _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - if (PyType_Check(val)) { - Type* t = PyInstance::extractTypeFrom(val); - if (t) { - visitor(t); - } - } - }); } template @@ -357,34 +333,8 @@ class FunctionOverload { visitor.visitHash(ShaHash(mGlobals.size())); - for (auto nameAndGlobal: mGlobals) { - if (!PyCell_Check(nameAndGlobal.second)) { - throw std::runtime_error( - "A global in mFunctionGlobalsInCells is somehow not a cell" - ); - } - - PyObject* o = PyCell_Get(nameAndGlobal.second); - Type* t = o && PyType_Check(o) ? PyInstance::extractTypeFrom(o) : nullptr; - - if (t && t->isForwardDefined()) { - if (t->isResolved()) { - visitor.visitNamedTopo( - nameAndGlobal.first, - t->forwardResolvesTo() - ); - } else { - // deliberately ignore non-resolved forwards - std::cout << "Deliberately ignoring " << nameAndGlobal.first << " -> " << TypeOrPyobj(t).name() << " since its not reoslved..\n"; - } - } else if (t) { - visitor.visitNamedTopo(nameAndGlobal.first, t); - } else { - visitor.visitNamedTopo( - nameAndGlobal.first, - nameAndGlobal.second - ); - } + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second._visitCompilerVisibleInternals(visitor); } if (mReturnType) { @@ -412,28 +362,12 @@ class FunctionOverload { } visitor.visitHash(ShaHash(1)); + visitor.visitHash(ShaHash(mGlobals.size())); - _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - Type* t = PyInstance::extractTypeFrom(val); - - if (t && t->isForwardDefined()) { - if (t->isResolved()) { - visitor.visitNamedTopo( - name, - t->forwardResolvesTo() - ); - } else { - // deliberately ignore non-resolved forwards - } - } else if (t) { - visitor.visitNamedTopo(name, t); - } else { - visitor.visitNamedTopo( - name, - val - ); - } - }); + for (auto& nameAndGlobal: mGlobals) { + visitor.visitName(nameAndGlobal.first); + nameAndGlobal.second._visitCompilerVisibleInternals(visitor); + } visitor.visitHash(ShaHash(1)); } @@ -502,11 +436,6 @@ class FunctionOverload { } } - template - void _visitCompilerVisibleGlobals(const visitor_type& visitor) { - visitCompilerVisibleGlobals(visitor, (PyCodeObject*)mFunctionCode, mFunctionGlobals); - } - template void _visitContainedTypes(const visitor_type& visitor) { } @@ -579,10 +508,6 @@ class FunctionOverload { return mSignatureFunction; } - PyObject* getFunctionGlobals() const { - return mFunctionGlobals; - } - const std::map& getGlobals() const { return mGlobals; } @@ -653,6 +578,10 @@ class FunctionOverload { }); } + // get a list of all "global dot accesses" contained in 'code'. This will pull out every + // case where we have a sequence of opcodes that access a global variable by name and then + // sequentially access members. So if you write 'x.y.z' and 'x' is a reference that will + // be looked up using a 'LOAD_GLOBAL' then we will include 'x.y.z' in outAccesses. static void extractGlobalAccessesFromCode(PyCodeObject* code, std::set& outAccesses) { uint8_t* bytes; Py_ssize_t bytecount; @@ -704,7 +633,7 @@ class FunctionOverload { void extractGlobalAccessesFromCodeIncludingCells( std::set& outNames ) const { - for (auto nameAndGlobal: mFunctionGlobalsInCells) { + for (auto nameAndGlobal: mGlobals) { outNames.insert(nameAndGlobal.first); } @@ -713,96 +642,28 @@ class FunctionOverload { extractGlobalAccessesFromCode((PyCodeObject*)mFunctionCode, outNames); } - void setGlobals(PyObject* globals) { - decref(mFunctionGlobals); - mFunctionGlobals = incref(globals); - } - bool symbolIsUnresolved(std::string name, bool insistForwardsResolved) { if (mClosureBindings.find(name) != mClosureBindings.end()) { return false; } - auto it = mFunctionGlobalsInCells.find(name); - if (it != mFunctionGlobalsInCells.end()) { - if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - if (insistForwardsResolved) { - PyObject* o = PyCell_Get(it->second); - Type* t = PyInstance::extractTypeFrom(o); - if (t && t->isForwardDefined()) { - return true; - } - } - - return false; - } + auto it = mGlobals.find(name); + if (it == mGlobals.end()) { return true; } - if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { - if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { - if (insistForwardsResolved) { - PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - Type* t = PyInstance::extractTypeFrom(o); - if (t && t->isForwardDefined()) { - return true; - } - } - - return false; - } - - PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); - if (builtins && PyDict_GetItemString(builtins, name.c_str())) { - return false; - } - } - - return true; + return it->second.isUnresolved(insistForwardsResolved); } void autoresolveGlobal( std::string name, const std::set& resolvedForwards ) { - auto it = mFunctionGlobalsInCells.find(name); - if (it != mFunctionGlobalsInCells.end()) { - if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - if (PyType_Check(PyCell_GET(it->second))) { - Type* cellContents = PyInstance::extractTypeFrom( - (PyTypeObject*)PyCell_Get(it->second) - ); - - if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - PyCell_Set( - it->second, - (PyObject*)PyInstance::typeObj( - cellContents->forwardResolvesTo() - ) - ); - } - } - } - - return; - } - - PyObject* dictVal = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - if (dictVal && PyType_Check(dictVal)) { - Type* cellContents = PyInstance::extractTypeFrom( - (PyTypeObject*)dictVal - ); + auto it = mGlobals.find(name); - if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - PyDict_SetItemString( - mFunctionGlobals, - name.c_str(), - (PyObject*)PyInstance::typeObj( - cellContents->forwardResolvesTo() - ) - ); - } + if (it != mGlobals.end()) { + it->second.autoresolveGlobal(resolvedForwards); } } @@ -843,73 +704,41 @@ class FunctionOverload { return ""; } - PyObject* getUsedGlobals() const { - // restrict the globals to contain only the values it references. - PyObject* result = PyDict_New(); - + static void buildInitialGlobalsDict( + std::map& outGlobals, + PyObject* inFuncGlobals, + PyCodeObject* inFuncCode + ) { std::set allNamesString; - extractGlobalAccessesFromCodeIncludingCells(allNamesString); + extractGlobalAccessesFromCode(inFuncCode, allNamesString); for (auto nameStr: allNamesString) { - if (nameStr != "__module_hash__" && mClosureBindings.find(nameStr) != mClosureBindings.end()) { + if (nameStr != "__module_hash__" && outGlobals.find(nameStr) != outGlobals.end()) { throw std::runtime_error( - "Somehow we have both a closure binding and a global for " - + nameStr + "Somehow we already a closure binding for " + nameStr + + " and somehow we want to register a global binding?" ); } - } - // iterate mFunctionGlobals, keeping any where the name is in allNames - // note we split on '.' and take the first part so that if a module - // like lxml.etree is included, and we use 'lxml', we'll take the reference - // to etree as well. This ensures that submodules in anonymously serialized - // code can get pulled along. - PyObject *key, *value; - Py_ssize_t pos = 0; - - // place them in sorted order - std::map > toCopy; - - while (PyDict_Next(mFunctionGlobals, &pos, &key, &value)) { - if (PyUnicode_Check(key)) { - std::string globalName = PyUnicode_AsUTF8(key); - std::string shortGlobalName = globalName; - - size_t indexOfDot = shortGlobalName.find('.'); - if (indexOfDot != std::string::npos) { - shortGlobalName = shortGlobalName.substr(0, indexOfDot); - } - - if (allNamesString.find(shortGlobalName) != allNamesString.end()) { - toCopy[globalName] = std::make_pair(key, value); - } - } - } - - for (auto& nameandKV: toCopy) { - if (PyDict_SetItem(result, nameandKV.second.first, nameandKV.second.second)) { - throw PythonExceptionSet(); - } - } + std::pair ref = FunctionGlobal::DottedGlobalsLookup( + inFuncGlobals, + nameStr + ); - PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); - if (builtins) { - PyDict_SetItemString(result, "__builtins__", builtins); + outGlobals[ref.first] = ref.second; } - - return result; } // create a new function object for this closure (or cache it // if we have no closure) - PyObject* buildFunctionObj(Type* closureType, instance_ptr closureData) const; + PyObject* buildFunctionObj(Type* closureType, instance_ptr closureData); FunctionOverload& operator=(const FunctionOverload& other) { other.increfAllPyObjects(); decrefAllPyObjects(); mFunctionCode = other.mFunctionCode; - mFunctionGlobals = other.mFunctionGlobals; + mGlobals = other.mGlobals; mFunctionDefaults = other.mFunctionDefaults; mFunctionAnnotations = other.mFunctionAnnotations; mSignatureFunction = other.mSignatureFunction; @@ -927,7 +756,6 @@ class FunctionOverload { mHasKwarg = other.mHasKwarg; mMinPositionalArgs = other.mMinPositionalArgs; mMaxPositionalArgs = other.mMaxPositionalArgs; - mFunctionGlobalsInCells = other.mFunctionGlobalsInCells; mCachedFunctionObj = other.mCachedFunctionObj; @@ -957,9 +785,9 @@ class FunctionOverload { buffer.writeBeginCompound(5); int varIx = 0; - for (auto nameAndCell: mFunctionGlobalsInCells) { - buffer.writeStringObject(varIx++, nameAndCell.first); - context.serializePythonObject(nameAndCell.second, buffer, varIx++); + for (auto& nameAndGlobal: mGlobals) { + buffer.writeStringObject(varIx++, nameAndGlobal.first); + nameAndGlobal.second.serialize(context, buffer, varIx++); } buffer.writeEndCompound(); @@ -998,20 +826,16 @@ class FunctionOverload { template static FunctionOverload deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { PyObjectHolder functionCode; - PyObjectHolder functionGlobals; PyObjectHolder functionAnnotations; PyObjectHolder functionDefaults; PyObjectHolder functionSignature; std::vector closureVarnames; - std::map functionGlobalsInCells; - std::map functionGlobalsInCellsRaw; + std::map functionGlobals; std::map closureBindings; Type* returnType = nullptr; Type* methodOf = nullptr; std::vector args; - functionGlobals.steal(PyDict_New()); - buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { functionCode.steal(context.deserializePythonObject(buffer, wireType)); @@ -1038,8 +862,7 @@ class FunctionOverload { if (last == "") { throw std::runtime_error("Corrupt Function closure encountered"); } - functionGlobalsInCells[last].steal(context.deserializePythonObject(buffer, wireType)); - functionGlobalsInCellsRaw[last] = functionGlobalsInCells[last]; + functionGlobals[last] = FunctionGlobal::deserialize(context, buffer, wireType); last = ""; } }); @@ -1087,7 +910,6 @@ class FunctionOverload { void increfAllPyObjects() const { incref(mFunctionCode); - incref(mFunctionGlobals); incref(mFunctionDefaults); incref(mFunctionAnnotations); incref(mSignatureFunction); @@ -1095,7 +917,6 @@ class FunctionOverload { void decrefAllPyObjects() { decref(mFunctionCode); - decref(mFunctionGlobals); decref(mFunctionDefaults); decref(mFunctionAnnotations); decref(mSignatureFunction); diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 06fab0b2c..c7a3e7c80 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -380,6 +380,10 @@ class Function : public Type { return mOverloads; } + std::vector& getOverloads() { + return mOverloads; + } + void addCompiledSpecialization( long whichOverload, compiled_code_entrypoint entrypoint, diff --git a/typed_python/ModuleRepresentationCopyContext.hpp b/typed_python/ModuleRepresentationCopyContext.hpp index 3e8005800..a034e85dc 100644 --- a/typed_python/ModuleRepresentationCopyContext.hpp +++ b/typed_python/ModuleRepresentationCopyContext.hpp @@ -226,10 +226,9 @@ class ModuleRepresentationCopyContext { overloads.push_back( FunctionOverload( o.getFunctionCode(), - copyObj(o.getFunctionGlobals()), copyObj(o.getFunctionDefaults()), copyObj(o.getFunctionAnnotations()), - o.getFunctionGlobalsInCells(), + o.getGlobals(), o.getFunctionClosureVarnames(), o.getClosureVariableBindings(), copyType(o.getReturnType()), @@ -654,12 +653,11 @@ class ModuleRepresentationCopyContext { } } - reachable.insert(PyObjectHandle(o.getFunctionGlobals())); - - for (auto nameAndObj: o.getFunctionGlobalsInCells()) { - if (nameAndObj.second) { - reachable.insert(PyObjectHandle(nameAndObj.second)); - } + for (auto nameAndGlobal: o.getGlobals()) { + throw std::runtime_error("ModuleRepresentationCopyContext doesn't know what to do with new globals yet"); + // if (nameAndObj.second) { + // reachable.insert(PyObjectHandle(nameAndObj.second)); + // } } if (o.getFunctionAnnotations()) { diff --git a/typed_python/PyFunctionInstance.cpp b/typed_python/PyFunctionInstance.cpp index 6ecaa5aac..7de62623f 100644 --- a/typed_python/PyFunctionInstance.cpp +++ b/typed_python/PyFunctionInstance.cpp @@ -35,7 +35,7 @@ PyObject* PyFunctionInstance::prepareArgumentToBePassedToCompiler(PyObject* o) { // static std::pair -PyFunctionInstance::tryToCallAnyOverload(const Function* f, instance_ptr funcClosure, PyObject* self, +PyFunctionInstance::tryToCallAnyOverload(Function* f, instance_ptr funcClosure, PyObject* self, PyObject* args, PyObject* kwargs) { //if we are an entrypoint, map any untyped function arguments to typed functions PyObjectHolder mappedArgs; @@ -97,7 +97,7 @@ PyFunctionInstance::tryToCallAnyOverload(const Function* f, instance_ptr funcClo // static std::pair PyFunctionInstance::tryToCallOverload( - const Function* f, + Function* f, instance_ptr functionClosure, long overloadIx, PyObject* self, @@ -105,8 +105,7 @@ std::pair PyFunctionInstance::tryToCallOverload( PyObject* kwargs, ConversionLevel conversionLevel ) { - const FunctionOverload& overload(f->getOverloads()[overloadIx]); - + FunctionOverload& overload(f->getOverloads()[overloadIx]); FunctionCallArgMapping mapping(overload); mapping.pushArguments(self, args, kwargs); @@ -192,11 +191,11 @@ std::pair PyFunctionInstance::tryToCallOverload( } std::pair PyFunctionInstance::getOverloadReturnType( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs ) { - const FunctionOverload& overload(f->getOverloads()[overloadIx]); + FunctionOverload& overload(f->getOverloads()[overloadIx]); Type* returnType = overload.getReturnType(); if (overload.getSignatureFunction()) { @@ -228,7 +227,7 @@ std::pair PyFunctionInstance::getOverloadReturnType( } std::pair PyFunctionInstance::determineReturnTypeForMatchedCall( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs, PyObject* self, @@ -356,7 +355,7 @@ std::pair PyFunctionInstance::determineReturnTypeForMatchedCall( return std::make_pair(actualReturnTypeAndCls.first, false); } -std::pair PyFunctionInstance::tryToCall(const Function* f, instance_ptr closure, PyObject* arg0, PyObject* arg1, PyObject* arg2) { +std::pair PyFunctionInstance::tryToCall(Function* f, instance_ptr closure, PyObject* arg0, PyObject* arg1, PyObject* arg2) { PyObjectStealer argTuple( (arg0 && arg1 && arg2) ? PyTuple_Pack(3, arg0, arg1, arg2) : @@ -371,12 +370,12 @@ std::pair PyFunctionInstance::tryToCall(const Function* f, inst // static std::pair PyFunctionInstance::dispatchFunctionCallToNative( - const Function* f, + Function* f, instance_ptr functionClosure, long overloadIx, const FunctionCallArgMapping& mapper ) { - const FunctionOverload& overload(f->getOverloads()[overloadIx]); + FunctionOverload& overload(f->getOverloads()[overloadIx]); for (const auto& spec: overload.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -420,7 +419,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( throw std::runtime_error("Somehow, the number of overloads in the function changed."); } - const FunctionOverload& overload2(convertedF->getOverloads()[overloadIx]); + FunctionOverload& overload2(convertedF->getOverloads()[overloadIx]); for (const auto& spec: overload2.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -457,7 +456,7 @@ std::pair PyFunctionInstance::dispatchFunctionCallToNative( decref(res); - const FunctionOverload& convertedOverload(convertedF->getOverloads()[overloadIx]); + FunctionOverload& convertedOverload(convertedF->getOverloads()[overloadIx]); for (const auto& spec: convertedOverload.getCompiledSpecializations()) { auto res = dispatchFunctionCallToCompiledSpecialization( @@ -568,10 +567,13 @@ PyObject* PyFunctionInstance::createOverloadPyRepresentation(Function* f) { PyObjectStealer pyIndex(PyLong_FromLong(k)); - PyObjectStealer pyGlobalCellDict(PyDict_New()); + PyObjectStealer pyFunctionGlobals(PyDict_New()); - for (auto nameAndCell: overload.getFunctionGlobalsInCells()) { - PyDict_SetItemString(pyGlobalCellDict, nameAndCell.first.c_str(), nameAndCell.second); + for (auto nameAndGlobal: overload.getGlobals()) { + PyObject* val = nameAndGlobal.second.getValueAsPyobj(); + if (val) { + PyDict_SetItemString(pyFunctionGlobals, nameAndGlobal.first.c_str(), val); + } } PyObjectStealer pyClosureVarsDict(PyDict_New()); @@ -639,11 +641,10 @@ PyObject* PyFunctionInstance::createOverloadPyRepresentation(Function* f) { PyDict_SetItemString(pyOverloadInstDict, "index", (PyObject*)pyIndex); PyDict_SetItemString(pyOverloadInstDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); PyDict_SetItemString(pyOverloadInstDict, "functionCode", (PyObject*)overload.getFunctionCode()); - PyDict_SetItemString(pyOverloadInstDict, "funcGlobalsInCells", (PyObject*)pyGlobalCellDict); + PyDict_SetItemString(pyOverloadInstDict, "functionGlobals", (PyObject*)pyFunctionGlobals); PyDict_SetItemString(pyOverloadInstDict, "returnType", overload.getReturnType() ? (PyObject*)typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); PyDict_SetItemString(pyOverloadInstDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); PyDict_SetItemString(pyOverloadInstDict, "methodOf", overload.getMethodOf() ? (PyObject*)typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "_realizedGlobals", Py_None); PyDict_SetItemString(pyOverloadInstDict, "args", argsTup); PyTuple_SetItem(overloadTuple, k, incref(pyOverloadInst)); @@ -779,35 +780,6 @@ PyObject* PyFunctionInstance::extractPyFun(PyObject* funcObj, PyObject* args, Py }); } -/* static */ -PyObject* PyFunctionInstance::extractOverloadGlobals(PyObject* cls, PyObject* args, PyObject* kwargs) { - Type* selfType = PyInstance::unwrapTypeArgToTypePtr(cls); - - if (!selfType || selfType->getTypeCategory() != Type::TypeCategory::catFunction) { - PyErr_Format(PyExc_TypeError, "Expected class to be a Function"); - return nullptr; - } - - Function* fType = (Function*)selfType; - - static const char *kwlist[] = {"overloadIx", NULL}; - - long overloadIx; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "l", (char**)kwlist, &overloadIx)) { - return nullptr; - } - - if (overloadIx < 0 || overloadIx >= fType->getOverloads().size()) { - PyErr_SetString(PyExc_IndexError, "Overload index out of bounds"); - return nullptr; - } - - return translateExceptionToPyObject([&]() { - return incref(fType->getOverloads()[overloadIx].getFunctionGlobals()); - }); -} - /* static */ PyObject* PyFunctionInstance::getClosure(PyObject* funcObj, PyObject* args, PyObject* kwargs) { static const char *kwlist[] = {NULL}; @@ -1002,7 +974,6 @@ PyMethodDef* PyFunctionInstance::typeMethodsConcrete(Type* t) { {"typeWithNocompile", (PyCFunction)PyFunctionInstance::typeWithNocompile, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, {"resultTypeFor", (PyCFunction)PyFunctionInstance::resultTypeFor, METH_VARARGS | METH_KEYWORDS, NULL}, {"extractPyFun", (PyCFunction)PyFunctionInstance::extractPyFun, METH_VARARGS | METH_KEYWORDS, NULL}, - {"extractOverloadGlobals", (PyCFunction)PyFunctionInstance::extractOverloadGlobals, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, {"getClosure", (PyCFunction)PyFunctionInstance::getClosure, METH_VARARGS | METH_KEYWORDS, NULL}, {"withClosureType", (PyCFunction)PyFunctionInstance::withClosureType, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, {"withOverloadVariableBindings", (PyCFunction)PyFunctionInstance::withOverloadVariableBindings, METH_VARARGS | METH_KEYWORDS | METH_CLASS, NULL}, diff --git a/typed_python/PyFunctionInstance.hpp b/typed_python/PyFunctionInstance.hpp index 5f56fe796..76290e536 100644 --- a/typed_python/PyFunctionInstance.hpp +++ b/typed_python/PyFunctionInstance.hpp @@ -30,13 +30,13 @@ class PyFunctionInstance : public PyInstance { static void copyConstructFromPythonInstanceConcrete(modeled_type* type, instance_ptr tgt, PyObject* pyRepresentation, ConversionLevel level); - static std::pair tryToCall(const Function* f, instance_ptr functionClosure, PyObject* arg0=nullptr, PyObject* arg1=nullptr, PyObject* arg2=nullptr); + static std::pair tryToCall(Function* f, instance_ptr functionClosure, PyObject* arg0=nullptr, PyObject* arg1=nullptr, PyObject* arg2=nullptr); - static std::pair tryToCallAnyOverload(const Function* f, instance_ptr functionClosure, PyObject* self, PyObject* args, PyObject* kwargs); + static std::pair tryToCallAnyOverload(Function* f, instance_ptr functionClosure, PyObject* self, PyObject* args, PyObject* kwargs); // determine the exact return type of a specific overload. Returns static std::pair getOverloadReturnType( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs ); @@ -45,7 +45,7 @@ class PyFunctionInstance : public PyInstance { // this calls signature functions, and checks for return type consistency. // Returns static std::pair determineReturnTypeForMatchedCall( - const Function* f, + Function* f, long overloadIx, FunctionCallArgMapping& matchedArgs, PyObject* self, @@ -60,7 +60,7 @@ class PyFunctionInstance : public PyInstance { static PyObject* prepareArgumentToBePassedToCompiler(PyObject* o); static std::pair tryToCallOverload( - const Function* f, + Function* f, instance_ptr funcClosure, long overloadIx, PyObject* self, @@ -74,7 +74,7 @@ class PyFunctionInstance : public PyInstance { //if 'isEntrypoint', then if we don't match a compiled specialization, ask the runtime to produce //one for us. static std::pair dispatchFunctionCallToNative( - const Function* f, + Function* f, instance_ptr functionClosure, long overloadIx, const FunctionCallArgMapping& mapping @@ -109,8 +109,6 @@ class PyFunctionInstance : public PyInstance { static PyObject* extractPyFun(PyObject* funcObj, PyObject* args, PyObject* kwargs); - static PyObject* extractOverloadGlobals(PyObject* funcObj, PyObject* args, PyObject* kwargs); - static PyObject* typeWithEntrypoint(PyObject* cls, PyObject* args, PyObject* kwargs); static PyObject* typeWithNocompile(PyObject* cls, PyObject* args, PyObject* kwargs); diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 3fb4e2df3..4fd208453 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1086,31 +1086,6 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur } } - // fill out the globals of function objects. we have to deserialize these - // in a separate final pass because they can have references to existing - // native types, and we need those objects to be filled out fully. - b.consumeCompoundMessage(wireType, [&](size_t indexInGroup, size_t subWireType) { - b.consumeCompoundMessage(subWireType, [&](size_t overloadIx, size_t subSubWireType) { - PyObjectStealer overloadGlobals(deserializePythonObject(b, subSubWireType)); - - if (actuallyBuildGroup) { - if (!indicesOfNativeTypes[indexInGroup] || - !indicesOfNativeTypes[indexInGroup]->isFunction()) { - throw std::runtime_error("Invalid function overload globals: not a function"); - } - - Function* f = (Function*)indicesOfNativeTypes[indexInGroup]; - if (overloadIx >= f->getOverloads().size()) { - throw std::runtime_error( - "Invalid function overload globals: overload index out of bounds" - ); - } - - ((FunctionOverload*)&f->getOverloads()[overloadIx])->setGlobals(overloadGlobals); - } - }); - }); - // now that we've set function globals, we need to go over all the classes and // held classes and make sure the versions of the function types they're actually // using have the new definitions. Unfortunately, Overload objects are not pointers diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 365a35074..954c9ae3c 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -811,30 +811,6 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec b.writeEndCompound(); -# if TP_VERBOSE_SERIALIZE - std::cout << s.prefix() << "Serializing used globals\n"; -# endif - - b.writeBeginCompound(4); - - for (auto& indexAndObj: group->getIndexToObject()) { - if (indexAndObj.second.type() && indexAndObj.second.type()->isFunction()) { - // we need to serialize the 'globals' dict of each function object - Function* f = (Function*)indexAndObj.second.type(); - - b.writeBeginCompound(indexAndObj.first); - - for (long k = 0; k < f->getOverloads().size(); k++) { - PyObjectStealer globals(f->getOverloads()[k].getUsedGlobals()); - serializePythonObject(globals, b, k); - } - - b.writeEndCompound(); - } - } - - b.writeEndCompound(); - b.writeEndCompound(); } diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 15ab829e7..bfb9a44a4 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -1096,8 +1096,8 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { std::vector closureVarnames; std::vector closureVarTypes; - std::map globalsInCells; std::map closureBindings; + std::map globals; PyObject* closure = PyFunction_GetClosure(funcObj); @@ -1135,7 +1135,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } - globalsInCells[closureVarnames[ix]] = incref(cell); + globals[closureVarnames[ix]] = FunctionGlobal::FromCell(cell); } } else { @@ -1148,13 +1148,18 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { } } + FunctionOverload::buildInitialGlobalsDict( + globals, + PyFunction_GetGlobals(funcObj), + (PyCodeObject*)PyFunction_GetCode(funcObj) + ); + overloads.push_back( FunctionOverload( PyFunction_GetCode(funcObj), - PyFunction_GetGlobals(funcObj), PyFunction_GetDefaults(funcObj), ((PyFunctionObject*)(PyObject*)funcObj)->func_annotations, - globalsInCells, + globals, closureVarnames, closureBindings, rType, diff --git a/typed_python/internals.py b/typed_python/internals.py index 6c23059ea..5a66cb816 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -428,15 +428,16 @@ def __repr__(self): class FunctionOverload: - def __init__(self, functionTypeObject, index, code, funcGlobalsInCells, closureVarLookups, returnType, signatureFunction, methodOf): + def __init__(self, functionTypeObject, index, code, functionGlobals, closureVarLookups, returnType, signatureFunction, methodOf): """Initialize a FunctionOverload. Args: functionTypeObject - a _types.Function type object representing the function index - the index within the _types.Function sequence of overloads we represent code - the code object for the function we're wrapping - funcGlobals - the globals for the function we're wrapping - funcGlobalsInCells - a dict of cells that also act like globals + functionGlobals - the globals for the function we're wrapping. These will have been + resolved by the forward type resolver, and so this dict may not be the same + as the actual function globals we started with. closureVarLookups - a dict from str to a list of ClosureVariableBindingSteps indicating how each function's closure variables are found in the closure of the actual function. @@ -450,17 +451,12 @@ def __init__(self, functionTypeObject, index, code, funcGlobalsInCells, closureV self.index = index self.closureVarLookups = closureVarLookups self.functionCode = code - self.funcGlobalsInCells = funcGlobalsInCells + self.functionGlobals = functionGlobals self.returnType = returnType self.signatureFunction = signatureFunction - self._realizedGlobals = None self.methodOf = methodOf self.args = () - @property - def functionGlobals(self): - return self.functionTypeObject.extractOverloadGlobals(self.index) - @staticmethod def extractGlobalNamesFromCode(codeObj): res = set() @@ -472,31 +468,6 @@ def extractGlobalNamesFromCode(codeObj): return res - @property - def realizedGlobals(self): - """Merge the 'functionGlobals' and the set of globals in 'cells' into a single dict.""" - if self._realizedGlobals is None: - res = {} - - globalNames = set(self.extractGlobalNamesFromCode(self.functionCode)) - - for name in self.functionGlobals: - if name.split(".")[0] in globalNames: - res[name] = self.functionGlobals[name] - - if '__module_hash__' in self.functionGlobals: - res['__module_hash__'] = self.functionGlobals['__module_hash__'] - - for varname, cell in self.funcGlobalsInCells.items(): - res[varname] = cell.cell_contents - - if self.methodOf is not None: - res['__class__'] = self.methodOf.Class - - self._realizedGlobals = res - - return self._realizedGlobals - @property def name(self): return self.functionTypeObject.__name__ From 8b024e50891dc36b1762b2c5ff9737dc84e15aa0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 29 Jun 2023 20:40:06 +0000 Subject: [PATCH 40/83] New FunctionGlobals kind of working? --- typed_python/CompilerVisibleObjectVisitor.hpp | 16 +- typed_python/FunctionGlobal.cpp | 48 ++++ typed_python/FunctionGlobal.hpp | 254 ++++++++---------- typed_python/__init__.py | 16 +- typed_python/_types.cpp | 26 +- typed_python/all.cpp | 1 + typed_python/type_construction_test.py | 4 +- 7 files changed, 191 insertions(+), 174 deletions(-) create mode 100644 typed_python/FunctionGlobal.cpp diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index afa894acb..30f21fd55 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -146,15 +146,21 @@ class LambdaVisitor { const visitor_2& nameVisit, const visitor_3& topoVisitor, const visitor_4& namedVisitor, - const visitor_5& onErr + const visitor_5& onErr, + VisibilityType visType ): mHashVisit(hashVisit), mNameVisit(nameVisit), mTopoVisitor(topoVisitor), mNamedVisitor(namedVisitor), - mOnErr(onErr) + mOnErr(onErr), + mVisType(visType) {} + VisibilityType visibilityType() const { + return mVisType; + } + void visitHash(ShaHash h) const { mHashVisit(h); } @@ -234,6 +240,8 @@ class LambdaVisitor { const visitor_3& mTopoVisitor; const visitor_4& mNamedVisitor; const visitor_5& mOnErr; + + VisibilityType mVisType; }; @@ -278,7 +286,7 @@ class CompilerVisibleObjectVisitor { obj, visibility, LambdaVisitor( - hashVisit, nameVisit, topoVisitor, namedVisitor, onErr + hashVisit, nameVisit, topoVisitor, namedVisitor, onErr, visibility ) ); } @@ -557,7 +565,7 @@ class CompilerVisibleObjectVisitor { walk(obj, visibility, LambdaVisitor( - hashVisit, nameVisit, topoVisitor, namedVisitor, onErr + hashVisit, nameVisit, topoVisitor, namedVisitor, onErr, visibility ) ); } diff --git a/typed_python/FunctionGlobal.cpp b/typed_python/FunctionGlobal.cpp new file mode 100644 index 000000000..d7aa45095 --- /dev/null +++ b/typed_python/FunctionGlobal.cpp @@ -0,0 +1,48 @@ +#include "FunctionGlobal.hpp" +#include "CompilerVisibleObjectVisitor.hpp" + + +/* static */ +std::pair FunctionGlobal::DottedGlobalsLookup( + PyObject* funcGlobals, + std::string dotSequence +) { + if (!PyDict_Check(funcGlobals)) { + throw std::runtime_error("Can't handle a non-dict function globals"); + } + + std::string shortGlobalName = dotSequence; + + size_t indexOfDot = shortGlobalName.find('.'); + if (indexOfDot != std::string::npos) { + shortGlobalName = shortGlobalName.substr(0, indexOfDot); + } + + PyObject* globalVal = PyDict_GetItemString(funcGlobals, shortGlobalName.c_str()); + + if (!globalVal) { + PyObject* builtins = PyDict_GetItemString(funcGlobals, "__builtins__"); + + if (builtins && PyDict_Check(builtins) && PyDict_GetItemString(builtins, shortGlobalName.c_str())) { + return DottedGlobalsLookup(builtins, dotSequence); + } + } + + std::string moduleName = CompilerVisibleObjectVisitor::isPyObjectGloballyIdentifiableModuleDict(funcGlobals); + if (moduleName.size()) { + return std::make_pair( + shortGlobalName, + FunctionGlobal::NamedModuleMember( + funcGlobals, + moduleName, + shortGlobalName + ) + ); + } + + // TODO: use the remainder of the dot sequence usefully + return std::make_pair( + shortGlobalName, + FunctionGlobal::GlobalInDict(funcGlobals, shortGlobalName) + ); +} diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index d419bcdc9..4aa2c3516 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -25,19 +25,21 @@ class FunctionGlobal { Unbound = 1, // this global is not bound to anything - its a Name error NamedModuleMember = 2, // this global is a member of a particular module Constant = 3, // this global is bound to a constant that we resolved to at some point - ForwardInDict = 4, // this global is an unresolved variable residing in some python dict - ForwardInCell = 5, // this global is an unresolved variable residing in some PyCell + GlobalInDict = 4, // this global is an unresolved variable residing in some python dict + GlobalInCell = 5, // this global is an unresolved variable residing in some PyCell }; FunctionGlobal( GlobalType inKind, PyObject* dictOrCell, std::string name, + std::string moduleName, CompilerVisiblePyObj* constant ) : mKind(inKind), mModuleDictOrCell(incref(dictOrCell)), mName(name), + mModuleName(name), mConstant(constant) { } @@ -58,6 +60,7 @@ class FunctionGlobal { GlobalType::Constant, nullptr, "", + "", constant ); } @@ -68,47 +71,9 @@ class FunctionGlobal { static std::pair DottedGlobalsLookup( PyObject* funcGlobals, std::string dotSequence - ) { - if (!PyDict_Check(funcGlobals)) { - throw std::runtime_error("Can't handle a non-dict function globals"); - } - - std::string shortGlobalName = dotSequence; - - size_t indexOfDot = shortGlobalName.find('.'); - if (indexOfDot != std::string::npos) { - shortGlobalName = shortGlobalName.substr(0, indexOfDot); - } - - PyObject* globalVal = PyDict_GetItemString(funcGlobals, shortGlobalName.c_str()); - - if (!globalVal) { - PyObject* builtins = PyDict_GetItemString(funcGlobals, "__builtins__"); - if (builtins && PyDict_Check(builtins) && PyDict_GetItemString(builtins, shortGlobalName.c_str())) { - return std::make_pair( - shortGlobalName, - FunctionGlobal::NamedModuleMember(builtins, shortGlobalName) - ); - } - } - - // TODO: this should be more precise - we need to decide what to do if - // this is a named global in a globally visible module vs a module - // still being defined, or whatever - return std::make_pair( - shortGlobalName, - FunctionGlobal::ForwardInDict(funcGlobals, shortGlobalName) - ); - } - - // construct a Global from a cell. If the cell is defined, we can resolve immediately - static FunctionGlobal FromCell(PyObject* cell) { - // pass - throw std::runtime_error("FunctionGlobal::FromCell not implemented yet"); - } - + ); - static FunctionGlobal NamedModuleMember(PyObject* moduleDict, std::string name) { + static FunctionGlobal NamedModuleMember(PyObject* moduleDict, std::string moduleName, std::string name) { if (!PyDict_Check(moduleDict)) { throw std::runtime_error("NamedModuleMember requires a Python dict object"); } @@ -117,32 +82,35 @@ class FunctionGlobal { GlobalType::NamedModuleMember, moduleDict, name, + "", nullptr ); } - static FunctionGlobal ForwardInDict(PyObject* dict, std::string name) { + static FunctionGlobal GlobalInDict(PyObject* dict, std::string name) { if (!PyDict_Check(dict)) { - throw std::runtime_error("ForwardInDict requires a Python dict object"); + throw std::runtime_error("GlobalInDict requires a Python dict object"); } return FunctionGlobal( - GlobalType::ForwardInDict, + GlobalType::GlobalInDict, dict, name, + "", nullptr ); } - static FunctionGlobal ForwardInCell(PyObject* cell, std::string name) { + static FunctionGlobal GlobalInCell(PyObject* cell) { if (!PyCell_Check(cell)) { - throw std::runtime_error("ForwardInCell requires a Python cell object"); + throw std::runtime_error("GlobalInCell requires a Python cell object"); } return FunctionGlobal( - GlobalType::ForwardInCell, + GlobalType::GlobalInCell, cell, - name, + "", + "", nullptr ); } @@ -159,12 +127,12 @@ class FunctionGlobal { return mKind == GlobalType::Constant; } - bool isForwardInDict() const { - return mKind == GlobalType::ForwardInDict; + bool isGlobalInDict() const { + return mKind == GlobalType::GlobalInDict; } - bool isForwardInCell() const { - return mKind == GlobalType::ForwardInCell; + bool isGlobalInCell() const { + return mKind == GlobalType::GlobalInCell; } PyObject* getValueAsPyobj() { @@ -187,6 +155,23 @@ class FunctionGlobal { return mModuleDictOrCell; } + PyObject* extractGlobalRefFromDictOrCell() { + if (isGlobalInDict()) { + return PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); + } + + if (isGlobalInCell()) { + return PyCell_Get(mModuleDictOrCell); + } + + if (isNamedModuleMember()) { + return PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); + } + + return nullptr; + } + + template void _visitReferencedTypes(const visitor_type& visitor) { if (isUnbound()) { @@ -198,130 +183,102 @@ class FunctionGlobal { return; } - //TODO: what should we do here? - return; + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* obj = extractGlobalRefFromDictOrCell(); + + if (obj) { + _visitReferencedTypesInPyobj(obj, visitor); + } + return; + } } template - void _visitCompilerVisibleInternals(const visitor_type& visitor) { - if (isUnbound()) { + void _visitReferencedTypesInPyobj(PyObject* obj, visitor_type vis) { + if (!obj) { return; } - if (isNamedModuleMember()) { - // TODO: need to know whether we're an identity or compiler visitor? + if (!PyType_Check(obj)) { return; } - if (isConstant()) { - mConstant->_visitCompilerVisibleInternals(visitor); - return; + Type* t = PyInstance::extractTypeFrom(obj); + + if (t) { + vis(t); } + } - if (isForwardInDict()) { + template + void _visitCompilerVisibleInternals(const visitor_type& visitor) { + if (isUnbound()) { return; } - if (isForwardInCell()) { + if (isConstant()) { + mConstant->_visitCompilerVisibleInternals(visitor); return; } - // if (!PyCell_Check(nameAndGlobal.second)) { - // throw std::runtime_error( - // "A global in mFunctionGlobalsInCells is somehow not a cell" - // ); - // } - - // PyObject* o = PyCell_Get(nameAndGlobal.second); - // Type* t = o && PyType_Check(o) ? PyInstance::extractTypeFrom(o) : nullptr; - - // if (t && t->isForwardDefined()) { - // if (t->isResolved()) { - // visitor.visitNamedTopo( - // nameAndGlobal.first, - // t->forwardResolvesTo() - // ); - // } else { - // // deliberately ignore non-resolved forwards - // std::cout << "Deliberately ignoring " << nameAndGlobal.first << " -> " << TypeOrPyobj(t).name() << " since its not reoslved..\n"; - // } - // } else if (t) { - // visitor.visitNamedTopo(nameAndGlobal.first, t); - // } else { - // visitor.visitNamedTopo( - // nameAndGlobal.first, - // nameAndGlobal.second - // ); - // } - + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* obj = extractGlobalRefFromDictOrCell(); + if (obj) { + _visitCompilerVisibleInternalsInPyobj(obj, visitor); + } + return; + } + } - // or alternatively - // - // _visitCompilerVisibleGlobals([&](const std::string& name, PyObject* val) { - // Type* t = PyInstance::extractTypeFrom(val); + template + void _visitCompilerVisibleInternalsInPyobj(PyObject* obj, const visitor_type& visitor) { + if (!PyType_Check(obj)) { + return; + } - // if (t && t->isForwardDefined()) { - // if (t->isResolved()) { - // visitor.visitNamedTopo( - // name, - // t->forwardResolvesTo() - // ); - // } else { - // // deliberately ignore non-resolved forwards - // } - // } else if (t) { - // visitor.visitNamedTopo(name, t); - // } else { - // visitor.visitNamedTopo( - // name, - // val - // ); - // } - // }); + Type* t = PyInstance::extractTypeFrom(obj); + if (t && t->isForwardDefined()) { + if (t->isResolved()) { + visitor.visitTopo( + t->forwardResolvesTo() + ); + } else { + } + } else if (t) { + visitor.visitTopo(t); + } else { + visitor.visitTopo(obj); + } } bool isUnresolved(bool insistForwardsResolved) { - throw std::runtime_error("FunctionGlobal::isUnresolved not implemented yet"); - - // auto it = mFunctionGlobalsInCells.find(name); - // if (it != mFunctionGlobalsInCells.end()) { - // if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - // if (insistForwardsResolved) { - // PyObject* o = PyCell_Get(it->second); - // Type* t = PyInstance::extractTypeFrom(o); - // if (t && t->isForwardDefined()) { - // return true; - // } - // } - - // return false; - // } + if (isConstant()) { + return false; + } + if (isUnbound()) { + return false; + } - // return true; - // } + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* obj = extractGlobalRefFromDictOrCell(); - // if (mFunctionGlobals && PyDict_Check(mFunctionGlobals)) { - // if (PyDict_GetItemString(mFunctionGlobals, name.c_str())) { - // if (insistForwardsResolved) { - // PyObject* o = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - // Type* t = PyInstance::extractTypeFrom(o); - // if (t && t->isForwardDefined()) { - // return true; - // } - // } + if (!obj) { + return true; + } - // return false; - // } + if (insistForwardsResolved) { + Type* t = PyInstance::extractTypeFrom(obj); + if (t && t->isForwardDefined()) { + return true; + } + } - // PyObject* builtins = PyDict_GetItemString(mFunctionGlobals, "__builtins__"); - // if (builtins && PyDict_GetItemString(builtins, name.c_str())) { - // return false; - // } - // } + return false; + } - // return true; + return true; } void autoresolveGlobal( @@ -419,5 +376,6 @@ class FunctionGlobal { CompilerVisiblePyObj* mConstant; std::string mName; + std::string mModuleName; PyObject* mModuleDictOrCell; }; diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 6aad85446..f20fa97db 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -75,17 +75,17 @@ _types._enableTypeAutoresolution(True) -from typed_python.SerializationContext import SerializationContext # noqa -from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa +# from typed_python.SerializationContext import SerializationContext # noqa +# from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa -from typed_python.generator import Generator # noqa +# from typed_python.generator import Generator # noqa -# this has to come at the end to break import cyclic -from typed_python.lib.map import map # noqa -from typed_python.lib.pmap import pmap # noqa -from typed_python.lib.reduce import reduce # noqa +# # this has to come at the end to break import cyclic +# from typed_python.lib.map import map # noqa +# from typed_python.lib.pmap import pmap # noqa +# from typed_python.lib.reduce import reduce # noqa -_types.initializeGlobalStatics() +# _types.initializeGlobalStatics() # start a background thread to release the GIL for us. Instead of immediately releasing the GIL, # we prefer to release it a short time after our C code no longer needs it, in case it diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index bfb9a44a4..f8f130c86 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -675,18 +675,20 @@ PyObject* initializeGlobalStatics(PyObject* nullValue, PyObject* args, PyObject* return NULL; } - // initialize all global references to modules. - // we can't initialize these lazily, because it may happen when we are - // doing something that prevents importing modules, like handling - // an exception - internalsModule(); - runtimeModule(); - builtinsModule(); - sysModule(); - osModule(); - weakrefModule(); + return translateExceptionToPyObject([&]() { + // initialize all global references to modules. + // we can't initialize these lazily, because it may happen when we are + // doing something that prevents importing modules, like handling + // an exception + internalsModule(); + runtimeModule(); + builtinsModule(); + sysModule(); + osModule(); + weakrefModule(); - return incref(Py_None); + return incref(Py_None); + }); } PyObject* isValidArithmeticConversion(PyObject* nullValue, PyObject* args, PyObject* kwargs) @@ -1135,7 +1137,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } - globals[closureVarnames[ix]] = FunctionGlobal::FromCell(cell); + globals[closureVarnames[ix]] = FunctionGlobal::GlobalInCell(cell); } } else { diff --git a/typed_python/all.cpp b/typed_python/all.cpp index a3c69de52..c001d08fa 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -62,6 +62,7 @@ compile the entire group all at once. #include "Type.cpp" #include "ClosureVariableBinding.cpp" #include "FunctionOverload.cpp" +#include "FunctionGlobal.cpp" #include "ForwardType.cpp" #include "ValueType.cpp" diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 755175ed3..f04887244 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,10 +3,10 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled + Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held#, NotCompiled ) -from typed_python.test_util import CodeEvaluator +# from typed_python.test_util import CodeEvaluator def test_identity_hash_of_alternative_stable(): From f58724d14902ebe744563fdff3be29e3fb36be36 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 30 Jun 2023 00:24:42 +0000 Subject: [PATCH 41/83] Full (but untested) resolution and copy pathway --- typed_python/CompilerVisiblePyObj.hpp | 10 +++ typed_python/FunctionGlobal.hpp | 114 +++++++++++++++++--------- typed_python/FunctionOverload.hpp | 26 ++++++ typed_python/FunctionType.hpp | 8 +- 4 files changed, 115 insertions(+), 43 deletions(-) diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index d7eaad899..7c705fcee 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -112,6 +112,16 @@ class CompilerVisiblePyObj { } + // get the python object representation of this object, which isn't guaranteed + // to exist and may need to be constructed on demand. + PyObject* getPyObj() { + if (mKind == Kind::Type) { + return (PyObject*)PyInstance::typeObj(mType); + } + + throw std::runtime_error("Can't make a python object representation for this pyobj"); + } + private: Kind mKind; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 4aa2c3516..3f987fae9 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -135,8 +135,29 @@ class FunctionGlobal { return mKind == GlobalType::GlobalInCell; } + Type* getValueAsType() { + PyObject* obj = getValueAsPyobj(); + if (obj) { + return PyInstance::extractTypeFrom(obj); + } + return nullptr; + } + PyObject* getValueAsPyobj() { - throw std::runtime_error("FunctionGlobal::getValueAsPyobj not implemented"); + if (isGlobalInCell() || isGlobalInCell() || isNamedModuleMember()) { + return extractGlobalRefFromDictOrCell(); + } + + if (isUnbound()) { + throw std::runtime_error("Unbound globals don't have python values."); + } + + if (isConstant()) { + return mConstant->getPyObj(); + } + + + throw std::runtime_error("Unknown global kind."); } GlobalType getKind() const { @@ -253,6 +274,22 @@ class FunctionGlobal { } } + FunctionGlobal withUpdatedInternalTypePointers(const std::map& groupMap) { + Type* t = getValueAsType(); + + auto it = groupMap.find(t); + + if (it != groupMap.end()) { + // we're actually a constant! + return FunctionGlobal::Constant( + CompilerVisiblePyObj::Type(it->second) + ); + } + + return *this; + } + + bool isUnresolved(bool insistForwardsResolved) { if (isConstant()) { return false; @@ -284,45 +321,42 @@ class FunctionGlobal { void autoresolveGlobal( const std::set& resolvedForwards ) { - throw std::runtime_error("FunctionGlobal::autoresolveGlobal not implemented yet"); - - // if (it != mFunctionGlobalsInCells.end()) { - // if (PyCell_Check(it->second) && PyCell_GET(it->second)) { - // if (PyType_Check(PyCell_GET(it->second))) { - // Type* cellContents = PyInstance::extractTypeFrom( - // (PyTypeObject*)PyCell_Get(it->second) - // ); - - // if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - // PyCell_Set( - // it->second, - // (PyObject*)PyInstance::typeObj( - // cellContents->forwardResolvesTo() - // ) - // ); - // } - // } - // } - - // return; - // } - - // PyObject* dictVal = PyDict_GetItemString(mFunctionGlobals, name.c_str()); - // if (dictVal && PyType_Check(dictVal)) { - // Type* cellContents = PyInstance::extractTypeFrom( - // (PyTypeObject*)dictVal - // ); - - // if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { - // PyDict_SetItemString( - // mFunctionGlobals, - // name.c_str(), - // (PyObject*)PyInstance::typeObj( - // cellContents->forwardResolvesTo() - // ) - // ); - // } - // } + if (isGlobalInCell()) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)PyCell_Get(mModuleDictOrCell) + ); + + if (cellContents && resolvedForwards.find(cellContents) != resolvedForwards.end()) { + PyCell_Set( + mModuleDictOrCell, + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + + return; + } + + if (isGlobalInDict() || isNamedModuleMember()) { + PyObject* dictVal = PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); + + if (dictVal) { + Type* cellContents = PyInstance::extractTypeFrom( + (PyTypeObject*)dictVal + ); + + if (resolvedForwards.find(cellContents) != resolvedForwards.end()) { + PyDict_SetItemString( + mModuleDictOrCell, + mName.c_str(), + (PyObject*)PyInstance::typeObj( + cellContents->forwardResolvesTo() + ) + ); + } + } + } } template diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 8db7dd001..b7d4a1167 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -263,6 +263,32 @@ class FunctionOverload { } } + void updateInternalTypePointers(const std::map& groupMap) { + if (mReturnType) { + Type::updateTypeRefFromGroupMap(mReturnType, groupMap); + } + + for (auto& a: mArgs) { + a._visitReferencedTypes([&](Type*& typePtr) { + Type::updateTypeRefFromGroupMap(typePtr, groupMap); + }); + } + + for (auto& varnameAndBinding: mClosureBindings) { + varnameAndBinding.second._visitReferencedTypes([&](Type*& typePtr) { + Type::updateTypeRefFromGroupMap(typePtr, groupMap); + }); + } + + if (mMethodOf) { + Type::updateTypeRefFromGroupMap(mMethodOf, groupMap); + } + + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second = nameAndGlobal.second.withUpdatedInternalTypePointers(groupMap); + } + } + template void _visitReferencedTypes(const visitor_type& visitor) { // we need to keep mArgs and mReturnType in sync with diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index c7a3e7c80..ed6f7f318 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -159,9 +159,11 @@ class Function : public Type { } void updateInternalTypePointersConcrete(const std::map& groupMap) { - _visitReferencedTypes([&](Type*& typePtr) { - updateTypeRefFromGroupMap(typePtr, groupMap); - }); + updateTypeRefFromGroupMap(mClosureType, groupMap); + + for (auto& o: mOverloads) { + o.updateInternalTypePointers(groupMap); + } } static Function* Make( From 882644a57c48bb0475200ed86c3e98125dd1b5db Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 12 Jul 2023 10:34:43 +0000 Subject: [PATCH 42/83] Add shims for python wrappers for new function globals internals. --- typed_python/PyCompilerVisiblePyObj.cpp | 114 +++++++++++++++++++ typed_python/PyCompilerVisiblePyObj.hpp | 36 ++++++ typed_python/PyFunctionGlobal.cpp | 114 +++++++++++++++++++ typed_python/PyFunctionGlobal.hpp | 36 ++++++ typed_python/PyFunctionOverload.cpp | 141 ++++++++++++++++++++++++ typed_python/PyFunctionOverload.hpp | 38 +++++++ typed_python/_types.cpp | 21 +++- typed_python/all.cpp | 3 + 8 files changed, 501 insertions(+), 2 deletions(-) create mode 100644 typed_python/PyCompilerVisiblePyObj.cpp create mode 100644 typed_python/PyCompilerVisiblePyObj.hpp create mode 100644 typed_python/PyFunctionGlobal.cpp create mode 100644 typed_python/PyFunctionGlobal.hpp create mode 100644 typed_python/PyFunctionOverload.cpp create mode 100644 typed_python/PyFunctionOverload.hpp diff --git a/typed_python/PyCompilerVisiblePyObj.cpp b/typed_python/PyCompilerVisiblePyObj.cpp new file mode 100644 index 000000000..e0e58d111 --- /dev/null +++ b/typed_python/PyCompilerVisiblePyObj.cpp @@ -0,0 +1,114 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyCompilerVisiblePyObj.hpp" + + +PyDoc_STRVAR(PyCompilerVisiblePyObj_doc, + "A single overload of a Function type object.\n\n" +); + +PyMethodDef PyCompilerVisiblePyObj_methods[] = { + {NULL} /* Sentinel */ +}; + +/* static */ +void PyCompilerVisiblePyObj::dealloc(PyCompilerVisiblePyObj *self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(CompilerVisiblePyObj* g) { + PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)PyType_CompilerVisiblePyObj.tp_alloc(&PyType_CompilerVisiblePyObj, 0); + self->mPyobj = g; + return (PyObject*)self; +} + +/* static */ +PyObject* PyCompilerVisiblePyObj::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyCompilerVisiblePyObj* self; + + self = (PyCompilerVisiblePyObj*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mPyobj = nullptr; + } + + return (PyObject*)self; +} + +/* static */ +int PyCompilerVisiblePyObj::init(PyCompilerVisiblePyObj *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "CompilerVisiblePyObj cannot be initialized directly"); + return -1; +} + +PyTypeObject PyType_CompilerVisiblePyObj = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "CompilerVisiblePyObj", + .tp_basicsize = sizeof(PyCompilerVisiblePyObj), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyCompilerVisiblePyObj::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = 0, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = 0, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyCompilerVisiblePyObj_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyCompilerVisiblePyObj_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyCompilerVisiblePyObj::init, + .tp_alloc = 0, + .tp_new = PyCompilerVisiblePyObj::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyCompilerVisiblePyObj.hpp b/typed_python/PyCompilerVisiblePyObj.hpp new file mode 100644 index 000000000..50e310a4b --- /dev/null +++ b/typed_python/PyCompilerVisiblePyObj.hpp @@ -0,0 +1,36 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" + +class PyCompilerVisiblePyObj { +public: + PyObject_HEAD + + CompilerVisiblePyObj* mPyobj; + + static void dealloc(PyCompilerVisiblePyObj *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyCompilerVisiblePyObj(CompilerVisiblePyObj* o); + + static int init(PyCompilerVisiblePyObj *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_CompilerVisiblePyObj; diff --git a/typed_python/PyFunctionGlobal.cpp b/typed_python/PyFunctionGlobal.cpp new file mode 100644 index 000000000..5af0d7de2 --- /dev/null +++ b/typed_python/PyFunctionGlobal.cpp @@ -0,0 +1,114 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyFunctionGlobal.hpp" + + +PyDoc_STRVAR(PyFunctionGlobal_doc, + "A single overload of a Function type object.\n\n" +); + +PyMethodDef PyFunctionGlobal_methods[] = { + {NULL} /* Sentinel */ +}; + +/* static */ +void PyFunctionGlobal::dealloc(PyFunctionGlobal *self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyFunctionGlobal::newPyFunctionGlobal(FunctionGlobal* g) { + PyFunctionGlobal* self = (PyFunctionGlobal*)PyType_FunctionGlobal.tp_alloc(&PyType_FunctionGlobal, 0); + self->mGlobal = g; + return (PyObject*)self; +} + +/* static */ +PyObject* PyFunctionGlobal::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyFunctionGlobal* self; + + self = (PyFunctionGlobal*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mGlobal = nullptr; + } + + return (PyObject*)self; +} + +/* static */ +int PyFunctionGlobal::init(PyFunctionGlobal *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "FunctionGlobal cannot be initialized directly"); + return -1; +} + +PyTypeObject PyType_FunctionGlobal = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "FunctionGlobal", + .tp_basicsize = sizeof(PyFunctionGlobal), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyFunctionGlobal::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = 0, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = 0, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyFunctionGlobal_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyFunctionGlobal_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyFunctionGlobal::init, + .tp_alloc = 0, + .tp_new = PyFunctionGlobal::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyFunctionGlobal.hpp b/typed_python/PyFunctionGlobal.hpp new file mode 100644 index 000000000..b79f9f147 --- /dev/null +++ b/typed_python/PyFunctionGlobal.hpp @@ -0,0 +1,36 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" + +class PyFunctionGlobal { +public: + PyObject_HEAD + + FunctionGlobal* mGlobal; + + static void dealloc(PyFunctionGlobal *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyFunctionGlobal(FunctionGlobal* g); + + static int init(PyFunctionGlobal *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_FunctionGlobal; diff --git a/typed_python/PyFunctionOverload.cpp b/typed_python/PyFunctionOverload.cpp new file mode 100644 index 000000000..31523fd08 --- /dev/null +++ b/typed_python/PyFunctionOverload.cpp @@ -0,0 +1,141 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyFunctionOverload.hpp" + + +PyDoc_STRVAR(PyFunctionOverload_doc, + "A single overload of a Function type object.\n\n" +); + +PyMethodDef PyFunctionOverload_methods[] = { + {NULL} /* Sentinel */ +}; + +/* static */ +void PyFunctionOverload::dealloc(PyFunctionOverload *self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyFunctionOverload::newPyFunctionOverload(Function* f, int64_t overloadIndex) { + PyFunctionOverload* self = (PyFunctionOverload*)PyType_FunctionOverload.tp_alloc(&PyType_FunctionOverload, 0); + self->mFunction = f; + self->mOverloadIx = overloadIndex; + return (PyObject*)self; +} + +/* static */ +PyObject* PyFunctionOverload::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyFunctionOverload* self; + + self = (PyFunctionOverload*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mFunction = nullptr; + self->mOverloadIx = 0; + } + + return (PyObject*)self; +} + +/* static */ +int PyFunctionOverload::init(PyFunctionOverload *self, PyObject *args, PyObject *kwargs) +{ + static const char* kwlist[] = {"functionType", "overloadIx", NULL}; + + PyObject* funcTypeObj; + long index; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ol", (char**)kwlist, &funcTypeObj, &index)) { + return -1; + } + + return translateExceptionToPyObjectReturningInt([&]() { + Type* t = PyInstance::unwrapTypeArgToTypePtr(funcTypeObj); + + if (!t || !t->isFunction()) { + throw std::runtime_error("Expected 'functionType' to be a Function type object"); + } + + Function* f = (Function*)t; + + if (index < 0 || index >= f->getOverloads().size()) { + throw std::runtime_error("overloadIx is out of bounds"); + } + + self->mFunction = f; + self->mOverloadIx = index; + + return 0; + }); +} + +PyTypeObject PyType_FunctionOverload = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "FunctionOverload", + .tp_basicsize = sizeof(PyFunctionOverload), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyFunctionOverload::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = 0, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = 0, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyFunctionOverload_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyFunctionOverload_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyFunctionOverload::init, + .tp_alloc = 0, + .tp_new = PyFunctionOverload::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyFunctionOverload.hpp b/typed_python/PyFunctionOverload.hpp new file mode 100644 index 000000000..879df3557 --- /dev/null +++ b/typed_python/PyFunctionOverload.hpp @@ -0,0 +1,38 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" + +class PyFunctionOverload { +public: + PyObject_HEAD + + Function* mFunction; + + int64_t mOverloadIx; + + static void dealloc(PyFunctionOverload *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyFunctionOverload(Function* f, int64_t overloadIndex); + + static int init(PyFunctionOverload *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_FunctionOverload; diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index f8f130c86..1e6e02eef 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -1,6 +1,5 @@ - /****************************************************************************** - Copyright 2017-2019 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,6 +35,9 @@ #include "UnicodeProps.hpp" #include "PyTemporaryReferenceTracer.hpp" #include "PySlab.hpp" +#include "PyFunctionOverload.hpp" +#include "PyFunctionGlobal.hpp" +#include "PyCompilerVisiblePyObj.hpp" #include "PyModuleRepresentation.hpp" #include "_types.hpp" #include "CompilerVisibleObjectVisitor.hpp" @@ -3793,10 +3795,25 @@ PyInit__types(void) return NULL; } + if (PyType_Ready(&PyType_FunctionOverload) < 0) { + return NULL; + } + if (PyType_Ready(&PyType_ModuleRepresentation) < 0) { return NULL; } + if (PyType_Ready(&PyType_FunctionGlobal) < 0) { + return NULL; + } + + if (PyType_Ready(&PyType_CompilerVisiblePyObj) < 0) { + return NULL; + } + + PyModule_AddObject(module, "FunctionOverload", (PyObject*)incref(&PyType_FunctionOverload)); + PyModule_AddObject(module, "CompilerVisiblePyObj", (PyObject*)incref(&PyType_CompilerVisiblePyObj)); + PyModule_AddObject(module, "FunctionGlobal", (PyObject*)incref(&PyType_FunctionGlobal)); PyModule_AddObject(module, "Slab", (PyObject*)incref(&PyType_Slab)); PyModule_AddObject(module, "ModuleRepresentation", (PyObject*)incref(&PyType_ModuleRepresentation)); diff --git a/typed_python/all.cpp b/typed_python/all.cpp index c001d08fa..a0a281b27 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -76,6 +76,9 @@ compile the entire group all at once. #include "PyModuleRepresentation.cpp" #include "Slab.cpp" #include "PyTemporaryReferenceTracer.cpp" +#include "PyFunctionOverload.cpp" +#include "PyFunctionGlobal.cpp" +#include "PyCompilerVisiblePyObj.cpp" #include "lz4.c" #include "lz4frame.c" From c7872b62890c264943aaafcbf00d601196f0110d Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 12 Jul 2023 07:38:06 -0400 Subject: [PATCH 43/83] FunctionOverload is implemented in C++ now. --- typed_python/FunctionOverload.hpp | 48 +++++++ typed_python/PyCompilerVisiblePyObj.cpp | 10 ++ typed_python/PyFunctionGlobal.cpp | 70 +++++++++- typed_python/PyFunctionGlobal.hpp | 14 +- typed_python/PyFunctionInstance.cpp | 90 +------------ typed_python/PyFunctionOverload.cpp | 124 ++++++++++++++---- typed_python/PyFunctionOverload.hpp | 4 + .../compiler/expression_conversion_context.py | 21 ++- typed_python/compiler/runtime.py | 4 +- typed_python/internals.py | 85 ------------ 10 files changed, 266 insertions(+), 204 deletions(-) diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index b7d4a1167..d555c59ec 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -538,6 +538,54 @@ class FunctionOverload { return mGlobals; } + std::map& getGlobals() { + return mGlobals; + } + + std::string signatureStr() { + if (mMethodOf) { + return "FunctionOverload(" + + mMethodOf->name() + + ", returns " + (mReturnType ? mReturnType->name() : std::string("anything")) + + ", " + + argsStr() + + ")"; + } else { + return "FunctionOverload(" + "returns " + (mReturnType ? mReturnType->name() : std::string("anything")) + + ", " + + argsStr() + + ")"; + } + } + + std::string argsStr() { + std::ostringstream s; + + for (long i = 0; i < mArgs.size(); i++) { + if (i) { + s << ", "; + } + + if (mArgs[i].getIsStarArg()) { + s << "*"; + } + + s << mArgs[i].getName(); + + if (mArgs[i].getTypeFilter()) { + s << ": " << mArgs[i].getTypeFilter()->name(); + } + + // TODO: default values should really be held in CompilerVisiblePyObj + if (mArgs[i].getDefaultValue()) { + s << " = " << "..."; + } + } + + return s.str(); + } + /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. In cases where we write something like 'x.y.z' the compiler shouldn't have a reference to 'x', diff --git a/typed_python/PyCompilerVisiblePyObj.cpp b/typed_python/PyCompilerVisiblePyObj.cpp index e0e58d111..0c58ae169 100644 --- a/typed_python/PyCompilerVisiblePyObj.cpp +++ b/typed_python/PyCompilerVisiblePyObj.cpp @@ -32,8 +32,18 @@ void PyCompilerVisiblePyObj::dealloc(PyCompilerVisiblePyObj *self) } PyObject* PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(CompilerVisiblePyObj* g) { + static std::unordered_map memo; + + auto it = memo.find(g); + if (it != memo.end()) { + return incref(it->second); + } + PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)PyType_CompilerVisiblePyObj.tp_alloc(&PyType_CompilerVisiblePyObj, 0); self->mPyobj = g; + + memo[g] = (PyObject*)self; + return (PyObject*)self; } diff --git a/typed_python/PyFunctionGlobal.cpp b/typed_python/PyFunctionGlobal.cpp index 5af0d7de2..45247d7ce 100644 --- a/typed_python/PyFunctionGlobal.cpp +++ b/typed_python/PyFunctionGlobal.cpp @@ -22,18 +22,80 @@ PyDoc_STRVAR(PyFunctionGlobal_doc, ); PyMethodDef PyFunctionGlobal_methods[] = { + {"isUnresolved", (PyCFunction)PyFunctionGlobal::isUnresolved, METH_VARARGS | METH_KEYWORDS, NULL}, + {"getValue", (PyCFunction)PyFunctionGlobal::getValue, METH_VARARGS | METH_KEYWORDS, NULL}, {NULL} /* Sentinel */ }; + +PyObject* PyFunctionGlobal::getValue(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs) { + static const char* kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + return NULL; + } + + return translateExceptionToPyObject([&]() { + PyObject* result = self->getGlobal().getValueAsPyobj(); + if (!result) { + // TODO: should this throw a NameError? + throw std::runtime_error("Global is not resolved"); + } + return incref(result); + }); +} + +PyObject* PyFunctionGlobal::isUnresolved(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs) { + static const char* kwlist[] = {"insistForwardsResolved", NULL}; + + int insistForwardsResolved = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|p", (char**)kwlist, &insistForwardsResolved)) { + return NULL; + } + + return translateExceptionToPyObject([&]() { + return incref(self->getGlobal().isUnresolved(insistForwardsResolved) ? Py_True : Py_False); + }); +} + +FunctionGlobal& PyFunctionGlobal::getGlobal() { + if (!mFuncType) { + throw std::runtime_error("FunctionGlobal is missing a function type"); + } + + if (!mName) { + throw std::runtime_error("FunctionGlobal is missing a name"); + } + + if (mOverloadIx < 0 || mOverloadIx >= mFuncType->getOverloads().size()){ + throw std::runtime_error("FunctionGlobal overload is out of bounds"); + } + + FunctionOverload& o = mFuncType->getOverloads()[mOverloadIx]; + + auto it = o.getGlobals().find(*mName); + if (it == o.getGlobals().end()) { + throw std::runtime_error("FunctionGlobal has no global named " + *mName); + } + + return it->second; +} + /* static */ void PyFunctionGlobal::dealloc(PyFunctionGlobal *self) { + if (self->mName) { + delete self->mName; + } Py_TYPE(self)->tp_free((PyObject*)self); } -PyObject* PyFunctionGlobal::newPyFunctionGlobal(FunctionGlobal* g) { +PyObject* PyFunctionGlobal::newPyFunctionGlobal(Function* func, long overloadIx, std::string globalName) { PyFunctionGlobal* self = (PyFunctionGlobal*)PyType_FunctionGlobal.tp_alloc(&PyType_FunctionGlobal, 0); - self->mGlobal = g; + self->mFuncType = func; + self->mOverloadIx = overloadIx; + self->mName = new std::string(globalName); return (PyObject*)self; } @@ -45,7 +107,9 @@ PyObject* PyFunctionGlobal::new_(PyTypeObject *type, PyObject *args, PyObject *k self = (PyFunctionGlobal*)type->tp_alloc(type, 0); if (self != NULL) { - self->mGlobal = nullptr; + self->mFuncType = nullptr; + self->mOverloadIx = 0; + self->mName = nullptr; } return (PyObject*)self; diff --git a/typed_python/PyFunctionGlobal.hpp b/typed_python/PyFunctionGlobal.hpp index b79f9f147..ec474a1fe 100644 --- a/typed_python/PyFunctionGlobal.hpp +++ b/typed_python/PyFunctionGlobal.hpp @@ -22,15 +22,25 @@ class PyFunctionGlobal { public: PyObject_HEAD - FunctionGlobal* mGlobal; + Function* mFuncType; + + int64_t mOverloadIx; + + std::string* mName; + + FunctionGlobal& getGlobal(); static void dealloc(PyFunctionGlobal *self); static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); - static PyObject* newPyFunctionGlobal(FunctionGlobal* g); + static PyObject* newPyFunctionGlobal(Function* func, long overloadIx, std::string globalName); static int init(PyFunctionGlobal *self, PyObject *args, PyObject *kwargs); + + static PyObject* isUnresolved(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs); + + static PyObject* getValue(PyFunctionGlobal* self, PyObject* args, PyObject* kwargs); }; extern PyTypeObject PyType_FunctionGlobal; diff --git a/typed_python/PyFunctionInstance.cpp b/typed_python/PyFunctionInstance.cpp index 7de62623f..1bc5cd921 100644 --- a/typed_python/PyFunctionInstance.cpp +++ b/typed_python/PyFunctionInstance.cpp @@ -556,98 +556,10 @@ std::pair PyFunctionInstance::dispatchFunctionCallToCompiledSpe // static PyObject* PyFunctionInstance::createOverloadPyRepresentation(Function* f) { - PyObject* funcOverload = staticPythonInstance("typed_python.internals", "FunctionOverload"); - PyObject* closureVariableCellLookupSingleton = staticPythonInstance("typed_python.internals", "CellAccess"); - PyObject* funcOverloadArg = staticPythonInstance("typed_python.internals", "FunctionOverloadArg"); - PyObjectStealer overloadTuple(PyTuple_New(f->getOverloads().size())); for (long k = 0; k < f->getOverloads().size(); k++) { - auto& overload = f->getOverloads()[k]; - - PyObjectStealer pyIndex(PyLong_FromLong(k)); - - PyObjectStealer pyFunctionGlobals(PyDict_New()); - - for (auto nameAndGlobal: overload.getGlobals()) { - PyObject* val = nameAndGlobal.second.getValueAsPyobj(); - if (val) { - PyDict_SetItemString(pyFunctionGlobals, nameAndGlobal.first.c_str(), val); - } - } - - PyObjectStealer pyClosureVarsDict(PyDict_New()); - - for (auto nameAndClosureVar: overload.getClosureVariableBindings()) { - PyObjectStealer bindingObj(PyTuple_New(nameAndClosureVar.second.size())); - - for (long k = 0; k < nameAndClosureVar.second.size(); k++) { - ClosureVariableBindingStep step = nameAndClosureVar.second[k]; - - if (step.isFunction()) { - // recall that 'PyTuple_SetItem' steals a reference, so we need to incref it here - PyTuple_SetItem(bindingObj, k, incref((PyObject*)typePtrToPyTypeRepresentation(step.getFunction()))); - } else - if (step.isNamedField()) { - PyTuple_SetItem(bindingObj, k, PyUnicode_FromString(step.getNamedField().c_str())); - } else - if (step.isIndexedField()) { - PyTuple_SetItem(bindingObj, k, PyLong_FromLong(step.getIndexedField())); - } else - if (step.isCellAccess()) { - PyTuple_SetItem(bindingObj, k, incref(closureVariableCellLookupSingleton)); - } else { - throw std::runtime_error("Corrupt ClosureVariableBindingStep encountered"); - } - } - - PyDict_SetItemString(pyClosureVarsDict, nameAndClosureVar.first.c_str(), bindingObj); - } - - PyObjectStealer emptyTup(PyTuple_New(0)); - PyObjectStealer emptyDict(PyDict_New()); - - // note that we can't actually call into the Python interpreter during this call, - // because that can release the GIL and allow other threads to access our type - // object before it's done. - PyObjectStealer argsTup(PyTuple_New(f->getOverloads()[k].getArgs().size())); - - for (long argIx = 0; argIx < f->getOverloads()[k].getArgs().size(); argIx++) { - auto arg = f->getOverloads()[k].getArgs()[argIx]; - - PyObjectStealer pyArgInst( - ((PyTypeObject*)funcOverloadArg)->tp_new((PyTypeObject*)funcOverloadArg, emptyTup, emptyDict) - ); - - PyObjectStealer pyArgInstDict(PyObject_GenericGetDict(pyArgInst, nullptr)); - - PyObjectStealer pyName(PyUnicode_FromString(arg.getName().c_str())); - PyDict_SetItemString(pyArgInstDict, "name", pyName); - PyDict_SetItemString(pyArgInstDict, "defaultValue", arg.getDefaultValue() ? PyTuple_Pack(1, arg.getDefaultValue()) : Py_None); - PyDict_SetItemString(pyArgInstDict, "_typeFilter", arg.getTypeFilter() ? (PyObject*)typePtrToPyTypeRepresentation(arg.getTypeFilter()) : Py_None); - PyDict_SetItemString(pyArgInstDict, "isStarArg", arg.getIsStarArg() ? Py_True : Py_False); - PyDict_SetItemString(pyArgInstDict, "isKwarg", arg.getIsKwarg() ? Py_True : Py_False); - - PyTuple_SetItem(argsTup, argIx, incref(pyArgInst)); - } - - PyObjectStealer pyOverloadInst( - ((PyTypeObject*)funcOverload)->tp_new((PyTypeObject*)funcOverload, emptyTup, emptyDict) - ); - - PyObjectStealer pyOverloadInstDict(PyObject_GenericGetDict(pyOverloadInst, nullptr)); - - PyDict_SetItemString(pyOverloadInstDict, "functionTypeObject", typePtrToPyTypeRepresentation(f)); - PyDict_SetItemString(pyOverloadInstDict, "index", (PyObject*)pyIndex); - PyDict_SetItemString(pyOverloadInstDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); - PyDict_SetItemString(pyOverloadInstDict, "functionCode", (PyObject*)overload.getFunctionCode()); - PyDict_SetItemString(pyOverloadInstDict, "functionGlobals", (PyObject*)pyFunctionGlobals); - PyDict_SetItemString(pyOverloadInstDict, "returnType", overload.getReturnType() ? (PyObject*)typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "methodOf", overload.getMethodOf() ? (PyObject*)typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "args", argsTup); - - PyTuple_SetItem(overloadTuple, k, incref(pyOverloadInst)); + PyTuple_SetItem(overloadTuple, k, PyFunctionOverload::newPyFunctionOverload(f, k)); } return incref(overloadTuple); diff --git a/typed_python/PyFunctionOverload.cpp b/typed_python/PyFunctionOverload.cpp index 31523fd08..c6e0c22ce 100644 --- a/typed_python/PyFunctionOverload.cpp +++ b/typed_python/PyFunctionOverload.cpp @@ -33,8 +33,11 @@ void PyFunctionOverload::dealloc(PyFunctionOverload *self) PyObject* PyFunctionOverload::newPyFunctionOverload(Function* f, int64_t overloadIndex) { PyFunctionOverload* self = (PyFunctionOverload*)PyType_FunctionOverload.tp_alloc(&PyType_FunctionOverload, 0); + self->mFunction = f; self->mOverloadIx = overloadIndex; + self->initFields(); + return (PyObject*)self; } @@ -53,36 +56,113 @@ PyObject* PyFunctionOverload::new_(PyTypeObject *type, PyObject *args, PyObject return (PyObject*)self; } -/* static */ -int PyFunctionOverload::init(PyFunctionOverload *self, PyObject *args, PyObject *kwargs) -{ - static const char* kwlist[] = {"functionType", "overloadIx", NULL}; +void PyFunctionOverload::initFields() { + FunctionOverload& overload = mFunction->getOverloads()[mOverloadIx]; + + PyObjectStealer pyClosureVarsDict(PyDict_New()); + + // note that we can't actually call into the Python interpreter during this call, + // because that can release the GIL and allow other threads to access our type + // object before it's done. + PyObjectStealer argsTup(PyTuple_New(overload.getArgs().size())); + + PyObject* closureVariableCellLookupSingleton = staticPythonInstance("typed_python.internals", "CellAccess"); + PyObject* funcOverloadArg = staticPythonInstance("typed_python.internals", "FunctionOverloadArg"); + + PyObjectStealer pyFunctionGlobals(PyDict_New()); + + for (auto nameAndGlobal: overload.getGlobals()) { + PyObject* val = nameAndGlobal.second.getValueAsPyobj(); + if (val) { + PyDict_SetItemString( + pyFunctionGlobals, + nameAndGlobal.first.c_str(), + PyFunctionGlobal::newPyFunctionGlobal( + mFunction, + mOverloadIx, + nameAndGlobal.first + ) + ); + } + } - PyObject* funcTypeObj; - long index; + for (auto nameAndClosureVar: overload.getClosureVariableBindings()) { + PyObjectStealer bindingObj(PyTuple_New(nameAndClosureVar.second.size())); + + for (long k = 0; k < nameAndClosureVar.second.size(); k++) { + ClosureVariableBindingStep step = nameAndClosureVar.second[k]; + + if (step.isFunction()) { + // recall that 'PyTuple_SetItem' steals a reference, so we need to incref it here + PyTuple_SetItem(bindingObj, k, incref((PyObject*)PyInstance::typePtrToPyTypeRepresentation(step.getFunction()))); + } else + if (step.isNamedField()) { + PyTuple_SetItem(bindingObj, k, PyUnicode_FromString(step.getNamedField().c_str())); + } else + if (step.isIndexedField()) { + PyTuple_SetItem(bindingObj, k, PyLong_FromLong(step.getIndexedField())); + } else + if (step.isCellAccess()) { + PyTuple_SetItem(bindingObj, k, incref(closureVariableCellLookupSingleton)); + } else { + throw std::runtime_error("Corrupt ClosureVariableBindingStep encountered"); + } + } - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ol", (char**)kwlist, &funcTypeObj, &index)) { - return -1; + PyDict_SetItemString(pyClosureVarsDict, nameAndClosureVar.first.c_str(), bindingObj); } - return translateExceptionToPyObjectReturningInt([&]() { - Type* t = PyInstance::unwrapTypeArgToTypePtr(funcTypeObj); + PyObjectStealer emptyTup(PyTuple_New(0)); + PyObjectStealer emptyDict(PyDict_New()); - if (!t || !t->isFunction()) { - throw std::runtime_error("Expected 'functionType' to be a Function type object"); - } + for (long argIx = 0; argIx < overload.getArgs().size(); argIx++) { + auto arg = overload.getArgs()[argIx]; - Function* f = (Function*)t; + PyObjectStealer pyArgInst( + ((PyTypeObject*)funcOverloadArg)->tp_new((PyTypeObject*)funcOverloadArg, emptyTup, emptyDict) + ); - if (index < 0 || index >= f->getOverloads().size()) { - throw std::runtime_error("overloadIx is out of bounds"); - } + PyObjectStealer pyArgInstDict(PyObject_GenericGetDict(pyArgInst, nullptr)); + + PyObjectStealer pyName(PyUnicode_FromString(arg.getName().c_str())); + PyDict_SetItemString(pyArgInstDict, "name", pyName); + PyDict_SetItemString(pyArgInstDict, "defaultValue", arg.getDefaultValue() ? PyTuple_Pack(1, arg.getDefaultValue()) : Py_None); + PyDict_SetItemString(pyArgInstDict, "_typeFilter", arg.getTypeFilter() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(arg.getTypeFilter()) : Py_None); + PyDict_SetItemString(pyArgInstDict, "isStarArg", arg.getIsStarArg() ? Py_True : Py_False); + PyDict_SetItemString(pyArgInstDict, "isKwarg", arg.getIsKwarg() ? Py_True : Py_False); + + PyTuple_SetItem(argsTup, argIx, incref(pyArgInst)); + } + + PyObjectStealer pyOverloadInstDict(PyObject_GenericGetDict((PyObject*)this, nullptr)); + + PyDict_SetItemString(pyOverloadInstDict, "functionTypeObject", PyInstance::typePtrToPyTypeRepresentation(mFunction)); + PyDict_SetItemString(pyOverloadInstDict, "index", (PyObject*)PyLong_FromLong(mOverloadIx)); + + PyDict_SetItemString(pyOverloadInstDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); + PyDict_SetItemString(pyOverloadInstDict, "functionCode", (PyObject*)overload.getFunctionCode()); + PyDict_SetItemString(pyOverloadInstDict, "functionGlobals", (PyObject*)pyFunctionGlobals); + PyDict_SetItemString(pyOverloadInstDict, "returnType", overload.getReturnType() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); + PyDict_SetItemString(pyOverloadInstDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); + PyDict_SetItemString(pyOverloadInstDict, "methodOf", overload.getMethodOf() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); + PyDict_SetItemString(pyOverloadInstDict, "args", argsTup); + PyDict_SetItemString(pyOverloadInstDict, "name", PyUnicode_FromString(mFunction->name().c_str())); +} + +/* static */ +int PyFunctionOverload::init(PyFunctionOverload *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "FunctionOverload cannot be initialized directly"); + return -1; +} - self->mFunction = f; - self->mOverloadIx = index; +// static +PyObject* PyFunctionOverload::tp_repr(PyObject *selfObj) { + PyFunctionOverload* self = (PyFunctionOverload*)selfObj; - return 0; - }); + return PyUnicode_FromString( + self->mFunction->getOverloads()[self->mOverloadIx].signatureStr().c_str() + ); } PyTypeObject PyType_FunctionOverload = { @@ -99,7 +179,7 @@ PyTypeObject PyType_FunctionOverload = { .tp_getattr = 0, .tp_setattr = 0, .tp_as_async = 0, - .tp_repr = 0, + .tp_repr = PyFunctionOverload::tp_repr, .tp_as_number = 0, .tp_as_sequence = 0, .tp_as_mapping = 0, diff --git a/typed_python/PyFunctionOverload.hpp b/typed_python/PyFunctionOverload.hpp index 879df3557..f0ffe7f14 100644 --- a/typed_python/PyFunctionOverload.hpp +++ b/typed_python/PyFunctionOverload.hpp @@ -26,6 +26,10 @@ class PyFunctionOverload { int64_t mOverloadIx; + void initFields(); + + static PyObject* tp_repr(PyObject *self); + static void dealloc(PyFunctionOverload *self); static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); diff --git a/typed_python/compiler/expression_conversion_context.py b/typed_python/compiler/expression_conversion_context.py index 24cb5948d..b29bb2331 100644 --- a/typed_python/compiler/expression_conversion_context.py +++ b/typed_python/compiler/expression_conversion_context.py @@ -806,6 +806,23 @@ def call_function_pointer(self, funcPtr, args, returnType): native_ast.CallTarget.Pointer(expr=funcPtr.cast(nativeFunType.pointer())).call(*native_args) ) + @staticmethod + def minPositionalCount(args): + for i in range(len(args)): + a = args[i] + if a.defaultValue or a.isStarArg or a.isKwarg: + return i + return len(args) + + @staticmethod + def maxPositionalCount(args): + for i in range(len(args)): + a = args[i] + if a.isStarArg: + return None + return len(args) + + @staticmethod def mapFunctionArguments(functionOverload: FunctionOverload, args, kwargs) -> OneOf(str, ListOf(FunctionArgMapping)): """Figure out how to call 'functionOverload' with 'args' and 'kwargs'. @@ -831,8 +848,8 @@ def mapFunctionArguments(functionOverload: FunctionOverload, args, kwargs) -> On outArgs = ListOf(FunctionArgMapping)() curTargetIx = 0 - minPositional = functionOverload.minPositionalCount() - maxPositional = functionOverload.maxPositionalCount() + minPositional = ExpressionConversionContext.minPositionalCount(functionOverload.args) + maxPositional = ExpressionConversionContext.maxPositionalCount(functionOverload.args) consumedPositionalNames = set() diff --git a/typed_python/compiler/runtime.py b/typed_python/compiler/runtime.py index 2a45a85bb..314b3e8c0 100644 --- a/typed_python/compiler/runtime.py +++ b/typed_python/compiler/runtime.py @@ -346,7 +346,9 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments fp = self.converter.functionPointerByName(wrappingCallTargetName) - overload._installNativePointer( + typed_python._types.installNativeFunctionPointer( + functionType, + overloadIx, fp.fp, callTarget.output_type.typeRepresentation if callTarget.output_type is not None else type(None), [i.typeRepresentation for i in callTarget.input_types] diff --git a/typed_python/internals.py b/typed_python/internals.py index 5a66cb816..9a9caa1a1 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -427,91 +427,6 @@ def __repr__(self): return res -class FunctionOverload: - def __init__(self, functionTypeObject, index, code, functionGlobals, closureVarLookups, returnType, signatureFunction, methodOf): - """Initialize a FunctionOverload. - - Args: - functionTypeObject - a _types.Function type object representing the function - index - the index within the _types.Function sequence of overloads we represent - code - the code object for the function we're wrapping - functionGlobals - the globals for the function we're wrapping. These will have been - resolved by the forward type resolver, and so this dict may not be the same - as the actual function globals we started with. - closureVarLookups - a dict from str to a list of ClosureVariableBindingSteps indicating - how each function's closure variables are found in the closure of the - actual function. - returnType - the return type annotation, or None if None provided. (if None was - specified, that would be the NoneType) - signatureFunction - None, or the user-provided callable that determines the - return type as a function of the types of the arguments to the function. - methodOf - if we are a method of a class, the class object - """ - self.functionTypeObject = functionTypeObject - self.index = index - self.closureVarLookups = closureVarLookups - self.functionCode = code - self.functionGlobals = functionGlobals - self.returnType = returnType - self.signatureFunction = signatureFunction - self.methodOf = methodOf - self.args = () - - @staticmethod - def extractGlobalNamesFromCode(codeObj): - res = set() - res.update(codeObj.co_names) - - for const in codeObj.co_consts: - if isinstance(const, type(codeObj)): - res.update(FunctionOverload.extractGlobalNamesFromCode(const)) - - return res - - @property - def name(self): - return self.functionTypeObject.__name__ - - def minPositionalCount(self): - for i in range(len(self.args)): - a = self.args[i] - if a.defaultValue or a.isStarArg or a.isKwarg: - return i - return len(self.args) - - def maxPositionalCount(self): - for i in range(len(self.args)): - a = self.args[i] - if a.isStarArg: - return None - return len(self.args) - - def addArg(self, name, defaultVal, typeFilter, isStarArg, isKwarg): - self.args = self.args + (FunctionOverloadArg(name, defaultVal, typeFilter, isStarArg, isKwarg),) - - def __str__(self): - if self.methodOf: - return "FunctionOverload(%s, returns %s, %s)" % ( - self.methodOf.Class.__name__, - self.returnType if self.returnType is not None else 'anything', - self.args - ) - - return "FunctionOverload(returns %s, %s)" % ( - self.returnType if self.returnType is not None else 'anything', - self.args - ) - - def _installNativePointer(self, fp, returnType, argumentTypes): - typed_python._types.installNativeFunctionPointer( - self.functionTypeObject, - self.index, - fp, - returnType, - tuple(argumentTypes)[len(self.closureVarLookups):], - ) - - class DisableCompiledCode: def __init__(self): pass From 977e98a4754e2aaf7830ef493e3805ac33367f50 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 12 Jul 2023 15:16:41 +0000 Subject: [PATCH 44/83] Full model of globals and constants. Fix a bug in function rehydration. --- typed_python/CompilerVisiblePyObj.hpp | 32 ++++++++++ typed_python/FunctionGlobal.hpp | 30 ++++++++- typed_python/FunctionOverload.cpp | 4 +- typed_python/FunctionOverload.hpp | 48 ++------------ typed_python/PyCompilerVisiblePyObj.cpp | 56 +++++++++++++++- typed_python/PyCompilerVisiblePyObj.hpp | 2 + typed_python/PyFunctionGlobal.cpp | 72 ++++++++++++++++++++- typed_python/PyFunctionGlobal.hpp | 4 ++ typed_python/PyFunctionOverload.cpp | 85 +++++++++++++++++++------ typed_python/PyFunctionOverload.hpp | 6 ++ typed_python/type_construction_test.py | 17 +++++ 11 files changed, 285 insertions(+), 71 deletions(-) diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 7c705fcee..9570d64f2 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -45,6 +45,22 @@ class CompilerVisiblePyObj { } public: + bool isUninitialized() const { + return mKind == Kind::Uninitialized; + } + + bool isType() const { + return mKind == Kind::Type; + } + + bool isInstance() const { + return mKind == Kind::Instance; + } + + bool isPyTuple() const { + return mKind == Kind::PyTuple; + } + static CompilerVisiblePyObj* Type(Type* t) { CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); @@ -122,6 +138,22 @@ class CompilerVisiblePyObj { throw std::runtime_error("Can't make a python object representation for this pyobj"); } + std::string toString() { + if (mKind == Kind::Type) { + return "CompilerVisiblePyObj.Type(" + mType->name() + ")"; + } + + if (mKind == Kind::Instance) { + return "CompilerVisiblePyObj.Instance(" + mInstance.type()->name() + ")"; + } + + if (mKind == Kind::PyTuple) { + return "CompilerVisiblePyObj.PyTuple()"; + } + + throw std::runtime_error("Unknown CompilerVisiblePyObj Kind."); + } + private: Kind mKind; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 3f987fae9..e9284e0ed 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -51,6 +51,30 @@ class FunctionGlobal { { } + std::string toString() { + if (isUnbound()) { + return "FunctionGlobal.Unbound()"; + } + + if (isNamedModuleMember()) { + return "FunctionGlobal.NamedModuleMember(" + mModuleName + ", " + mName + ")"; + } + + if (isGlobalInCell()) { + return "FunctionGlobal.GlobalInCell()"; + } + + if (isGlobalInDict()) { + return "FunctionGlobal.GlobalInDict()"; + } + + if (isConstant()) { + return "FunctionGlobal.Constant(" + mConstant->toString() + ")"; + } + + throw std::runtime_error("Unknown FunctionGlobal Kind"); + } + static FunctionGlobal Unbound() { return FunctionGlobal(); } @@ -82,7 +106,7 @@ class FunctionGlobal { GlobalType::NamedModuleMember, moduleDict, name, - "", + moduleName, nullptr ); } @@ -172,6 +196,10 @@ class FunctionGlobal { return mName; } + const std::string& getModuleName() const { + return mModuleName; + } + PyObject* getModuleDictOrCell() const { return mModuleDictOrCell; } diff --git a/typed_python/FunctionOverload.cpp b/typed_python/FunctionOverload.cpp index e30cd6f06..6a0a93528 100644 --- a/typed_python/FunctionOverload.cpp +++ b/typed_python/FunctionOverload.cpp @@ -80,7 +80,7 @@ PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr clo PyTuple_SetItem( (PyObject*)closureTup, k, - incref(globalVal) + PyCell_New(globalVal) ); } } else { @@ -97,7 +97,7 @@ PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr clo PyTuple_SetItem( (PyObject*)closureTup, k, - incref( + PyCell_New( ((PyCellType*)bindingValue.type())->getPyObj(bindingValue.data()) ) ); diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index d555c59ec..f426c45e3 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -167,6 +167,10 @@ class FunctionOverload { str << "("; + if (mMethodOf) { + str << "method of " << mMethodOf->name() << ", "; + } + for (long k = 0; k < mArgs.size(); k++) { if (k) { str << ", "; @@ -542,50 +546,6 @@ class FunctionOverload { return mGlobals; } - std::string signatureStr() { - if (mMethodOf) { - return "FunctionOverload(" - + mMethodOf->name() - + ", returns " + (mReturnType ? mReturnType->name() : std::string("anything")) - + ", " - + argsStr() - + ")"; - } else { - return "FunctionOverload(" - "returns " + (mReturnType ? mReturnType->name() : std::string("anything")) - + ", " - + argsStr() - + ")"; - } - } - - std::string argsStr() { - std::ostringstream s; - - for (long i = 0; i < mArgs.size(); i++) { - if (i) { - s << ", "; - } - - if (mArgs[i].getIsStarArg()) { - s << "*"; - } - - s << mArgs[i].getName(); - - if (mArgs[i].getTypeFilter()) { - s << ": " << mArgs[i].getTypeFilter()->name(); - } - - // TODO: default values should really be held in CompilerVisiblePyObj - if (mArgs[i].getDefaultValue()) { - s << " = " << "..."; - } - } - - return s.str(); - } - /* walk over the opcodes in 'code' and extract all cases where we're accessing globals by name. In cases where we write something like 'x.y.z' the compiler shouldn't have a reference to 'x', diff --git a/typed_python/PyCompilerVisiblePyObj.cpp b/typed_python/PyCompilerVisiblePyObj.cpp index 0c58ae169..de8131fa1 100644 --- a/typed_python/PyCompilerVisiblePyObj.cpp +++ b/typed_python/PyCompilerVisiblePyObj.cpp @@ -41,8 +41,8 @@ PyObject* PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(CompilerVisiblePyObj PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)PyType_CompilerVisiblePyObj.tp_alloc(&PyType_CompilerVisiblePyObj, 0); self->mPyobj = g; - - memo[g] = (PyObject*)self; + + memo[g] = incref((PyObject*)self); return (PyObject*)self; } @@ -61,6 +61,56 @@ PyObject* PyCompilerVisiblePyObj::new_(PyTypeObject *type, PyObject *args, PyObj return (PyObject*)self; } +PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrName) { + PyCompilerVisiblePyObj* pyCVPO = (PyCompilerVisiblePyObj*)selfObj; + + return translateExceptionToPyObject([&] { + if (!PyUnicode_Check(attrName)) { + throw std::runtime_error("Expected a string for attribute name"); + } + + std::string attr(PyUnicode_AsUTF8(attrName)); + + CompilerVisiblePyObj* obj = pyCVPO->mPyobj; + + if (attr == "kind") { + if (obj->isUninitialized()) { + return PyUnicode_FromString("Uninitialized"); + } + + if (obj->isType()) { + return PyUnicode_FromString("Type"); + } + + if (obj->isInstance()) { + return PyUnicode_FromString("Instance"); + } + + if (obj->isPyTuple()) { + return PyUnicode_FromString("Py2Tuple"); + } + + throw std::runtime_error("Unknown CompilerVisiblePyObj Kind"); + } + + if (attr == "type" && obj->isType()) { + Type* t = obj->getType(); + + if (!t) { + throw std::runtime_error("Somehow we don't have a type"); + } + + return incref((PyObject*)PyInstance::typeObj(t)); + } + + if (attr == "instance" && obj->isInstance()) { + return PyInstance::fromInstance(obj->getInstance()); + } + + return PyObject_GenericGetAttr(selfObj, attrName); + }); +} + /* static */ int PyCompilerVisiblePyObj::init(PyCompilerVisiblePyObj *self, PyObject *args, PyObject *kwargs) { @@ -89,7 +139,7 @@ PyTypeObject PyType_CompilerVisiblePyObj = { .tp_hash = 0, .tp_call = 0, .tp_str = 0, - .tp_getattro = 0, + .tp_getattro = PyCompilerVisiblePyObj::tp_getattro, .tp_setattro = 0, .tp_as_buffer = 0, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/typed_python/PyCompilerVisiblePyObj.hpp b/typed_python/PyCompilerVisiblePyObj.hpp index 50e310a4b..63851f60d 100644 --- a/typed_python/PyCompilerVisiblePyObj.hpp +++ b/typed_python/PyCompilerVisiblePyObj.hpp @@ -24,6 +24,8 @@ class PyCompilerVisiblePyObj { CompilerVisiblePyObj* mPyobj; + static PyObject* tp_getattro(PyObject* selfObj, PyObject* attr); + static void dealloc(PyCompilerVisiblePyObj *self); static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); diff --git a/typed_python/PyFunctionGlobal.cpp b/typed_python/PyFunctionGlobal.cpp index 45247d7ce..c02c13da1 100644 --- a/typed_python/PyFunctionGlobal.cpp +++ b/typed_python/PyFunctionGlobal.cpp @@ -115,6 +115,74 @@ PyObject* PyFunctionGlobal::new_(PyTypeObject *type, PyObject *args, PyObject *k return (PyObject*)self; } +PyObject* PyFunctionGlobal::tp_repr(PyObject *selfObj) { + PyFunctionGlobal* self = (PyFunctionGlobal*)selfObj; + + return translateExceptionToPyObject([&]() { + return PyUnicode_FromString(self->getGlobal().toString().c_str()); + }); +} + +PyObject* PyFunctionGlobal::tp_getattro(PyObject* selfObj, PyObject* attrName) { + PyFunctionGlobal* pyFuncGlobal = (PyFunctionGlobal*)selfObj; + + return translateExceptionToPyObject([&] { + if (!PyUnicode_Check(attrName)) { + throw std::runtime_error("Expected a string for attribute name"); + } + + std::string attr(PyUnicode_AsUTF8(attrName)); + + auto& global = pyFuncGlobal->getGlobal(); + + if (attr == "kind") { + if (global.isUnbound()) { + return PyUnicode_FromString("Unbound"); + } + + if (global.isNamedModuleMember()) { + return PyUnicode_FromString("NamedModuleMember"); + } + + if (global.isGlobalInCell()) { + return PyUnicode_FromString("GlobalInCell"); + } + + if (global.isGlobalInDict()) { + return PyUnicode_FromString("GlobalInDict"); + } + + if (global.isConstant()) { + return PyUnicode_FromString("Constant"); + } + + throw std::runtime_error("Unknown Global Kind"); + } + + if (attr == "constant" && global.isConstant()) { + return PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(global.getConstant()); + } + + if (attr == "name" && (global.isNamedModuleMember() || global.isGlobalInDict())) { + return PyUnicode_FromString(global.getName().c_str()); + } + + if (attr == "moduleName" && global.isNamedModuleMember()) { + return PyUnicode_FromString(global.getModuleName().c_str()); + } + + if (attr == "cell" && global.isGlobalInCell()) { + return incref(global.getModuleDictOrCell()); + } + + if (attr == "moduleDict" && (global.isNamedModuleMember() || global.isGlobalInDict())) { + return incref(global.getModuleDictOrCell()); + } + + return PyObject_GenericGetAttr(selfObj, attrName); + }); +} + /* static */ int PyFunctionGlobal::init(PyFunctionGlobal *self, PyObject *args, PyObject *kwargs) { @@ -136,14 +204,14 @@ PyTypeObject PyType_FunctionGlobal = { .tp_getattr = 0, .tp_setattr = 0, .tp_as_async = 0, - .tp_repr = 0, + .tp_repr = PyFunctionGlobal::tp_repr, .tp_as_number = 0, .tp_as_sequence = 0, .tp_as_mapping = 0, .tp_hash = 0, .tp_call = 0, .tp_str = 0, - .tp_getattro = 0, + .tp_getattro = PyFunctionGlobal::tp_getattro, .tp_setattro = 0, .tp_as_buffer = 0, .tp_flags = Py_TPFLAGS_DEFAULT, diff --git a/typed_python/PyFunctionGlobal.hpp b/typed_python/PyFunctionGlobal.hpp index ec474a1fe..9267e969f 100644 --- a/typed_python/PyFunctionGlobal.hpp +++ b/typed_python/PyFunctionGlobal.hpp @@ -30,6 +30,10 @@ class PyFunctionGlobal { FunctionGlobal& getGlobal(); + static PyObject* tp_getattro(PyObject* selfObj, PyObject* attr); + + static PyObject* tp_repr(PyObject *selfObj); + static void dealloc(PyFunctionGlobal *self); static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); diff --git a/typed_python/PyFunctionOverload.cpp b/typed_python/PyFunctionOverload.cpp index c6e0c22ce..71e843a12 100644 --- a/typed_python/PyFunctionOverload.cpp +++ b/typed_python/PyFunctionOverload.cpp @@ -25,17 +25,31 @@ PyMethodDef PyFunctionOverload_methods[] = { {NULL} /* Sentinel */ }; +FunctionOverload& PyFunctionOverload::getOverload() { + if (!mFunction) { + throw std::runtime_error("FunctionOverload has an empty FunctionType"); + } + + if (mOverloadIx < 0 || mOverloadIx >= mFunction->getOverloads().size()) { + throw std::runtime_error("FunctionOverload overloadIx out of bounds"); + } + + return mFunction->getOverloads()[mOverloadIx]; +} + /* static */ void PyFunctionOverload::dealloc(PyFunctionOverload *self) { + decref(self->mDict); Py_TYPE(self)->tp_free((PyObject*)self); } PyObject* PyFunctionOverload::newPyFunctionOverload(Function* f, int64_t overloadIndex) { PyFunctionOverload* self = (PyFunctionOverload*)PyType_FunctionOverload.tp_alloc(&PyType_FunctionOverload, 0); - + self->mFunction = f; self->mOverloadIx = overloadIndex; + self->mDict = PyDict_New(); self->initFields(); return (PyObject*)self; @@ -51,13 +65,14 @@ PyObject* PyFunctionOverload::new_(PyTypeObject *type, PyObject *args, PyObject if (self != NULL) { self->mFunction = nullptr; self->mOverloadIx = 0; + self->mDict = PyDict_New(); } return (PyObject*)self; } void PyFunctionOverload::initFields() { - FunctionOverload& overload = mFunction->getOverloads()[mOverloadIx]; + FunctionOverload& overload = getOverload(); PyObjectStealer pyClosureVarsDict(PyDict_New()); @@ -68,22 +83,19 @@ void PyFunctionOverload::initFields() { PyObject* closureVariableCellLookupSingleton = staticPythonInstance("typed_python.internals", "CellAccess"); PyObject* funcOverloadArg = staticPythonInstance("typed_python.internals", "FunctionOverloadArg"); - + PyObjectStealer pyFunctionGlobals(PyDict_New()); for (auto nameAndGlobal: overload.getGlobals()) { - PyObject* val = nameAndGlobal.second.getValueAsPyobj(); - if (val) { - PyDict_SetItemString( - pyFunctionGlobals, - nameAndGlobal.first.c_str(), - PyFunctionGlobal::newPyFunctionGlobal( - mFunction, - mOverloadIx, - nameAndGlobal.first - ) - ); - } + PyDict_SetItemString( + pyFunctionGlobals, + nameAndGlobal.first.c_str(), + PyFunctionGlobal::newPyFunctionGlobal( + mFunction, + mOverloadIx, + nameAndGlobal.first + ) + ); } for (auto nameAndClosureVar: overload.getClosureVariableBindings()) { @@ -141,7 +153,7 @@ void PyFunctionOverload::initFields() { PyDict_SetItemString(pyOverloadInstDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); PyDict_SetItemString(pyOverloadInstDict, "functionCode", (PyObject*)overload.getFunctionCode()); - PyDict_SetItemString(pyOverloadInstDict, "functionGlobals", (PyObject*)pyFunctionGlobals); + PyDict_SetItemString(pyOverloadInstDict, "globals", (PyObject*)pyFunctionGlobals); PyDict_SetItemString(pyOverloadInstDict, "returnType", overload.getReturnType() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); PyDict_SetItemString(pyOverloadInstDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); PyDict_SetItemString(pyOverloadInstDict, "methodOf", overload.getMethodOf() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); @@ -161,10 +173,45 @@ PyObject* PyFunctionOverload::tp_repr(PyObject *selfObj) { PyFunctionOverload* self = (PyFunctionOverload*)selfObj; return PyUnicode_FromString( - self->mFunction->getOverloads()[self->mOverloadIx].signatureStr().c_str() + ("FunctionOverload(" + self->mFunction->getOverloads()[self->mOverloadIx].toString() + ")").c_str() ); } +PyObject* PyFunctionOverload::tp_getattro(PyObject *o, PyObject* attrName) { + PyFunctionOverload* pyFuncOverload = (PyFunctionOverload*)o; + + return translateExceptionToPyObject([&] { + if (!PyUnicode_Check(attrName)) { + throw std::runtime_error("Expected a string for attribute name"); + } + + std::string attr(PyUnicode_AsUTF8(attrName)); + + if (attr == "realizedGlobals") { + PyObject* pyFunctionGlobals = PyDict_New(); + + FunctionOverload& overload = pyFuncOverload->getOverload(); + + for (auto nameAndGlobal: overload.getGlobals()) { + PyObject* val = nameAndGlobal.second.getValueAsPyobj(); + + if (val) { + PyDict_SetItemString( + pyFunctionGlobals, + nameAndGlobal.first.c_str(), + val + ); + } + } + + return pyFunctionGlobals; + } + + return PyObject_GenericGetAttr(o, attrName); + }); +} + + PyTypeObject PyType_FunctionOverload = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "FunctionOverload", @@ -186,7 +233,7 @@ PyTypeObject PyType_FunctionOverload = { .tp_hash = 0, .tp_call = 0, .tp_str = 0, - .tp_getattro = 0, + .tp_getattro = PyFunctionOverload::tp_getattro, .tp_setattro = 0, .tp_as_buffer = 0, .tp_flags = Py_TPFLAGS_DEFAULT, @@ -204,7 +251,7 @@ PyTypeObject PyType_FunctionOverload = { .tp_dict = 0, .tp_descr_get = 0, .tp_descr_set = 0, - .tp_dictoffset = 0, + .tp_dictoffset = offsetof(PyFunctionOverload, mDict), .tp_init = (initproc) PyFunctionOverload::init, .tp_alloc = 0, .tp_new = PyFunctionOverload::new_, diff --git a/typed_python/PyFunctionOverload.hpp b/typed_python/PyFunctionOverload.hpp index f0ffe7f14..94bf6f19b 100644 --- a/typed_python/PyFunctionOverload.hpp +++ b/typed_python/PyFunctionOverload.hpp @@ -26,8 +26,14 @@ class PyFunctionOverload { int64_t mOverloadIx; + PyObject* mDict; + + FunctionOverload& getOverload(); + void initFields(); + static PyObject* tp_getattro(PyObject *o, PyObject* attrName); + static PyObject* tp_repr(PyObject *self); static void dealloc(PyFunctionOverload *self); diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index f04887244..e6687ccc6 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -35,6 +35,11 @@ def g(): def test_type_looks_resolvable_alternative(): A = Alternative("A_", X={}, f=lambda self: B) + # hide a reference to this forward in a tuple that the autoresolver can't see + AFwd = (A,) + + assert isForwardDefined(A) + assert typeLooksResolvable(A, unambiguously=False) assert not typeLooksResolvable(A, unambiguously=True) @@ -48,6 +53,18 @@ def test_type_looks_resolvable_alternative(): A = resolveForwardDefinedType(A) B = resolveForwardDefinedType(B) + bGlobal = A.f.overloads[0].globals['B'] + assert bGlobal.kind == "Constant" + assert bGlobal.constant.kind == "Type" + assert bGlobal.constant.type is B + + bFwdGlobal = AFwd[0].f.overloads[0].globals['B'] + assert bFwdGlobal.kind == "GlobalInCell" + assert bFwdGlobal.getValue() is B + + assert not isForwardDefined(A) + assert isForwardDefined(AFwd[0]) + assert A().f() is B assert B().g() is A From 152434d8850cba8ffec44e9d3e5d5300e7c70dba Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 13 Jul 2023 23:07:31 +0000 Subject: [PATCH 45/83] Re-enable core of TP's __init__. Type construction test passing. --- typed_python/CompilerVisiblePyObj.hpp | 12 ++++++++++++ typed_python/FunctionGlobal.hpp | 3 +-- typed_python/Type.cpp | 8 ++++---- typed_python/__init__.py | 16 ++++++++-------- .../compiler/expression_conversion_context.py | 4 ++-- typed_python/type_construction_test.py | 6 +++--- 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 9570d64f2..a81bcdf7c 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -125,7 +125,19 @@ class CompilerVisiblePyObj { template void _visitCompilerVisibleInternals(const visitor_type& visitor) { + if (mKind == Kind::Type) { + visitor.visitTopo(mType); + } + if (mKind == Kind::Instance) { + // TODO: what to do here? + throw std::runtime_error("TODO: CompilerVisiblePyObj::_visitCompilerVisibleInternals"); + } + + if (mKind == Kind::PyTuple) { + // TODO: what to do here? + throw std::runtime_error("TODO: CompilerVisiblePyObj::_visitCompilerVisibleInternals"); + } } // get the python object representation of this object, which isn't guaranteed diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index e9284e0ed..9647a70e1 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -168,7 +168,7 @@ class FunctionGlobal { } PyObject* getValueAsPyobj() { - if (isGlobalInCell() || isGlobalInCell() || isNamedModuleMember()) { + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { return extractGlobalRefFromDictOrCell(); } @@ -180,7 +180,6 @@ class FunctionGlobal { return mConstant->getPyObj(); } - throw std::runtime_error("Unknown global kind."); } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index cd131eae9..ae7837aac 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -508,10 +508,10 @@ void Type::attemptToResolve() { typeAndSource.first->finalizeType(); } - std::cout << "resolving group:\n"; - for (auto typeAndSource: resolutionSource) { - std::cout << " " << TypeOrPyobj(typeAndSource.first).name() << " from " << TypeOrPyobj(typeAndSource.second).name() << "\n"; - } + // std::cout << "resolving group:\n"; + // for (auto typeAndSource: resolutionSource) { + // std::cout << " " << TypeOrPyobj(typeAndSource.first).name() << " from " << TypeOrPyobj(typeAndSource.second).name() << "\n"; + // } // now internalize the types by their hash. For each type, we compute a hash // and then look to see if we've seen it before. We build a lookup table from diff --git a/typed_python/__init__.py b/typed_python/__init__.py index f20fa97db..6aad85446 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -75,17 +75,17 @@ _types._enableTypeAutoresolution(True) -# from typed_python.SerializationContext import SerializationContext # noqa -# from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa +from typed_python.SerializationContext import SerializationContext # noqa +from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa -# from typed_python.generator import Generator # noqa +from typed_python.generator import Generator # noqa -# # this has to come at the end to break import cyclic -# from typed_python.lib.map import map # noqa -# from typed_python.lib.pmap import pmap # noqa -# from typed_python.lib.reduce import reduce # noqa +# this has to come at the end to break import cyclic +from typed_python.lib.map import map # noqa +from typed_python.lib.pmap import pmap # noqa +from typed_python.lib.reduce import reduce # noqa -# _types.initializeGlobalStatics() +_types.initializeGlobalStatics() # start a background thread to release the GIL for us. Instead of immediately releasing the GIL, # we prefer to release it a short time after our C code no longer needs it, in case it diff --git a/typed_python/compiler/expression_conversion_context.py b/typed_python/compiler/expression_conversion_context.py index b29bb2331..07e098f22 100644 --- a/typed_python/compiler/expression_conversion_context.py +++ b/typed_python/compiler/expression_conversion_context.py @@ -23,7 +23,8 @@ from typed_python.compiler.type_wrappers.slice_type_object_wrapper import SliceWrapper from typed_python.compiler.conversion_level import ConversionLevel from typed_python.SerializationContext import SerializationContext -from typed_python.internals import makeFunctionType, FunctionOverload +from typed_python.internals import makeFunctionType +from typed_python._types import FunctionOverload from typed_python.compiler.function_stack_state import FunctionStackState from typed_python.compiler.python_object_representation import pythonObjectRepresentation from typed_python.compiler.python_object_representation import pythonObjectRepresentationType @@ -822,7 +823,6 @@ def maxPositionalCount(args): return None return len(args) - @staticmethod def mapFunctionArguments(functionOverload: FunctionOverload, args, kwargs) -> OneOf(str, ListOf(FunctionArgMapping)): """Figure out how to call 'functionOverload' with 'args' and 'kwargs'. diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index e6687ccc6..40cbac1c6 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,10 +3,10 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held#, NotCompiled + Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled ) -# from typed_python.test_util import CodeEvaluator +from typed_python.test_util import CodeEvaluator def test_identity_hash_of_alternative_stable(): @@ -122,7 +122,7 @@ class C(Class): def f(self): return C().g() - c = C() + C() def test_autoresolve_forwards_with_nonforwards(): From fce652551be45305d6bef1f92eb74817990e523c Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 14 Jul 2023 21:52:06 +0000 Subject: [PATCH 46/83] Implement serialization of new FunctionGlobals pathway. --- repro.py | 34 ++------ typed_python/CompilerVisiblePyObj.hpp | 87 +++++++++++++++++++ typed_python/FunctionGlobal.hpp | 84 ++++++++++++++++-- typed_python/FunctionOverload.cpp | 12 ++- typed_python/PyFunctionOverload.cpp | 38 +++++--- typed_python/PyFunctionOverload.hpp | 4 + ...onSerializationContext_deserialization.cpp | 2 + ...thonSerializationContext_serialization.cpp | 3 + .../compiler/python_to_native_converter.py | 4 +- 9 files changed, 215 insertions(+), 53 deletions(-) diff --git a/repro.py b/repro.py index 76b9e8935..fae9c7b12 100644 --- a/repro.py +++ b/repro.py @@ -1,47 +1,29 @@ import sys -from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final +from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final, Function, ListOf from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr def writer(): - Base = Forward("Base") - - @Base.define class Base(Class): - def blah(self) -> Base: + def f(self): return self - def f(self, x) -> int: - return x + 1 - - class Child(Base, Final): - def f(self, x) -> int: - return -1 - - aChild = Child() - - aChildBytes = SerializationContext().serialize(aChild) + bytesToWrite = SerializationContext().serialize(Base) with open("a.dat", "wb") as f: - f.write(aChildBytes) + f.write(bytesToWrite) def reader(): with open("a.dat", "rb") as f: - aChild = SerializationContext().deserialize(f.read()) - - # @Entrypoint - def callF(x): - return x.f(10) + x = SerializationContext().deserialize(f.read()) - assert callF(aChild) == -1 + print(x.__name__) + print(x.f.overloads[0].globals) + print(x().f().f()) if sys.argv[1:] == ['r']: reader() else: writer() - - - - diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index a81bcdf7c..2950a1729 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -87,6 +87,15 @@ class CompilerVisiblePyObj { return res; } + static CompilerVisiblePyObj* PyTuple(const std::vector& elts) { + CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + + res->mKind = Kind::PyTuple; + res->mElements = elts; + + return res; + } + void append(CompilerVisiblePyObj* elt) { if (mKind != Kind::PyTuple) { throw std::runtime_error("Expected a PyTuple"); @@ -166,6 +175,84 @@ class CompilerVisiblePyObj { throw std::runtime_error("Unknown CompilerVisiblePyObj Kind."); } + + + template + void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + + TODO: implement memoize! + + buffer.writeBeginCompound(fieldNumber); + buffer.writeRegisterType(0, (uint64_t)mKind); + + if (mKind == Kind::Type) { + context.serializeNativeType(mType, buffer, 1); + } else + if (mKind == Kind::Instance) { + buffer.writeBeginCompound(2); + context.serializeNativeType(mInstance.type(), buffer, 0); + mInstance.type()->serialize(mInstance.data(), buffer, 1); + buffer.writeEndCompound(); + } else + if (mKind == Kind::PyTuple) { + buffer.writeBeginCompound(3); + for (long i = 0; i < mElements.size(); i++) { + mElements[i]->serialize(context, buffer, i); + } + buffer.writeEndCompound(); + } + + buffer.writeEndCompound(); + } + + template + static CompilerVisiblePyObj* deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + int64_t kind = 0; + + ::Type* type = nullptr; + + std::vector vec; + ::Instance i; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + buffer.readRegisterType(&kind, wireType); + } else + if (fieldNumber == 1) { + type = context.deserializeNativeType(buffer, wireType); + } else + if (fieldNumber == 2) { + i = context.deserializeNativeInstance(buffer, wireType); + } else + if (fieldNumber == 3) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + vec.push_back(CompilerVisiblePyObj::deserialize(context, buffer, wireType)); + }); + } + }); + + if (kind == (int)Kind::Type) { + if (!type) { + throw std::runtime_error("Corrupt CompilerVisiblePyObj::Type"); + } + return CompilerVisiblePyObj::Type(type); + } + + if (kind == (int)Kind::Instance) { + return CompilerVisiblePyObj::Instance(i); + } + + if (kind == (int)Kind::PyTuple) { + return CompilerVisiblePyObj::PyTuple(vec); + } + + if (kind == (int)Kind::Uninitialized) { + return new CompilerVisiblePyObj(); + } + + throw std::runtime_error("Corrupt CompilerVisiblePyObj - invalid kind"); + } + private: Kind mKind; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 9647a70e1..a304c2a93 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -173,7 +173,7 @@ class FunctionGlobal { } if (isUnbound()) { - throw std::runtime_error("Unbound globals don't have python values."); + return nullptr; } if (isConstant()) { @@ -304,6 +304,10 @@ class FunctionGlobal { FunctionGlobal withUpdatedInternalTypePointers(const std::map& groupMap) { Type* t = getValueAsType(); + if (!t) { + return *this; + } + auto it = groupMap.find(t); if (it != groupMap.end()) { @@ -388,15 +392,83 @@ class FunctionGlobal { template void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { - // context.serializePythonObject(nameAndCell.second, buffer, varIx++); - throw std::runtime_error("FunctionGlobal::serialize not implemented yet"); + buffer.writeBeginCompound(fieldNumber); + buffer.writeRegisterType(0, (uint64_t)mKind); + + if (mKind == GlobalType::NamedModuleMember) { + context.serializePythonObject(mModuleDictOrCell, buffer, 1); + buffer.writeStringObject(2, mName); + buffer.writeStringObject(3, mModuleName); + } else + if (mKind == GlobalType::Constant) { + mConstant->serialize(context, buffer, 4); + } else + if (mKind == GlobalType::GlobalInDict) { + context.serializePythonObject(mModuleDictOrCell, buffer, 1); + buffer.writeStringObject(2, mName); + } else + if (mKind == GlobalType::GlobalInCell) { + context.serializePythonObject(mModuleDictOrCell, buffer, 1); + } + + buffer.writeEndCompound(); } template static FunctionGlobal deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { - throw std::runtime_error("FunctionGlobal::deserialize not implemented yet"); - // functionGlobalsInCells[last].steal(context.deserializePythonObject(buffer, wireType)); - // functionGlobalsInCellsRaw[last] = functionGlobalsInCells[last]; + uint64_t kind = 0; + std::string name, moduleName; + PyObjectHolder cellOrModule; + CompilerVisiblePyObj* constant = nullptr; + + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + if (fieldNumber == 0) { + buffer.readRegisterType(&kind, wireType); + } else + if (fieldNumber == 1) { + cellOrModule.steal(context.deserializePythonObject(buffer, wireType)); + } else + if (fieldNumber == 2) { + name = buffer.readStringObject(); + } else + if (fieldNumber == 3) { + moduleName = buffer.readStringObject(); + } else + if (fieldNumber == 4) { + constant = CompilerVisiblePyObj::deserialize(context, buffer, wireType); + } + }); + + if (kind == (int)GlobalType::Unbound) { + return FunctionGlobal::Unbound(); + } + if (kind == (int)GlobalType::NamedModuleMember) { + if (!cellOrModule || !name.size() || !moduleName.size()) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid NamedModuleMember"); + } + return FunctionGlobal::NamedModuleMember(cellOrModule, name, moduleName); + } + if (kind == (int)GlobalType::Constant) { + if (constant) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid constant"); + } + return FunctionGlobal::Constant(constant); + } + if (kind == (int)GlobalType::GlobalInDict) { + if (!cellOrModule || !name.size()) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid GlobalInDict"); + } + return FunctionGlobal::GlobalInDict(cellOrModule, name); + } + if (kind == (int)GlobalType::GlobalInCell) { + if (!cellOrModule) { + throw std::runtime_error("Corrupt FunctionGlobal - invalid cell"); + } + return FunctionGlobal::GlobalInCell((PyObject*)cellOrModule); + } + + asm("int3"); + throw std::runtime_error("Corrupt FunctionGlobal - invalid 'kind'"); } bool operator<(const FunctionGlobal& g) const { diff --git a/typed_python/FunctionOverload.cpp b/typed_python/FunctionOverload.cpp index 6a0a93528..cefb6ceb2 100644 --- a/typed_python/FunctionOverload.cpp +++ b/typed_python/FunctionOverload.cpp @@ -76,13 +76,11 @@ PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr clo if (globalIt != mGlobals.end()) { PyObject* globalVal = globalIt->second.getValueAsPyobj(); - if (globalVal) { - PyTuple_SetItem( - (PyObject*)closureTup, - k, - PyCell_New(globalVal) - ); - } + PyTuple_SetItem( + (PyObject*)closureTup, + k, + PyCell_New(globalVal) + ); } else { auto bindingIt = mClosureBindings.find(varname); if (bindingIt == mClosureBindings.end()) { diff --git a/typed_python/PyFunctionOverload.cpp b/typed_python/PyFunctionOverload.cpp index 71e843a12..51f20754e 100644 --- a/typed_python/PyFunctionOverload.cpp +++ b/typed_python/PyFunctionOverload.cpp @@ -50,7 +50,7 @@ PyObject* PyFunctionOverload::newPyFunctionOverload(Function* f, int64_t overloa self->mFunction = f; self->mOverloadIx = overloadIndex; self->mDict = PyDict_New(); - self->initFields(); + self->mIsInitialized = false; return (PyObject*)self; } @@ -66,12 +66,23 @@ PyObject* PyFunctionOverload::new_(PyTypeObject *type, PyObject *args, PyObject self->mFunction = nullptr; self->mOverloadIx = 0; self->mDict = PyDict_New(); + self->mIsInitialized = false; } return (PyObject*)self; } +void PyFunctionOverload::ensureInitialized() { + if (!mIsInitialized) { + initFields(); + } +} + void PyFunctionOverload::initFields() { + if (mIsInitialized) { + throw std::runtime_error("Can't initialize a PyFunctionOverload twice."); + } + FunctionOverload& overload = getOverload(); PyObjectStealer pyClosureVarsDict(PyDict_New()); @@ -146,19 +157,20 @@ void PyFunctionOverload::initFields() { PyTuple_SetItem(argsTup, argIx, incref(pyArgInst)); } - PyObjectStealer pyOverloadInstDict(PyObject_GenericGetDict((PyObject*)this, nullptr)); + PyObject* funcTypeObj = PyInstance::typePtrToPyTypeRepresentation(mFunction); + PyDict_SetItemString(mDict, "functionTypeObject", funcTypeObj); + PyDict_SetItemString(mDict, "index", (PyObject*)PyLong_FromLong(mOverloadIx)); - PyDict_SetItemString(pyOverloadInstDict, "functionTypeObject", PyInstance::typePtrToPyTypeRepresentation(mFunction)); - PyDict_SetItemString(pyOverloadInstDict, "index", (PyObject*)PyLong_FromLong(mOverloadIx)); + PyDict_SetItemString(mDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); + PyDict_SetItemString(mDict, "functionCode", (PyObject*)overload.getFunctionCode()); + PyDict_SetItemString(mDict, "globals", (PyObject*)pyFunctionGlobals); + PyDict_SetItemString(mDict, "returnType", overload.getReturnType() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); + PyDict_SetItemString(mDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); + PyDict_SetItemString(mDict, "methodOf", overload.getMethodOf() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); + PyDict_SetItemString(mDict, "args", argsTup); + PyDict_SetItemString(mDict, "name", PyUnicode_FromString(mFunction->name().c_str())); - PyDict_SetItemString(pyOverloadInstDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); - PyDict_SetItemString(pyOverloadInstDict, "functionCode", (PyObject*)overload.getFunctionCode()); - PyDict_SetItemString(pyOverloadInstDict, "globals", (PyObject*)pyFunctionGlobals); - PyDict_SetItemString(pyOverloadInstDict, "returnType", overload.getReturnType() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "signatureFunction", overload.getSignatureFunction() ? (PyObject*)overload.getSignatureFunction() : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "methodOf", overload.getMethodOf() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getMethodOf()) : Py_None); - PyDict_SetItemString(pyOverloadInstDict, "args", argsTup); - PyDict_SetItemString(pyOverloadInstDict, "name", PyUnicode_FromString(mFunction->name().c_str())); + mIsInitialized = true; } /* static */ @@ -180,6 +192,8 @@ PyObject* PyFunctionOverload::tp_repr(PyObject *selfObj) { PyObject* PyFunctionOverload::tp_getattro(PyObject *o, PyObject* attrName) { PyFunctionOverload* pyFuncOverload = (PyFunctionOverload*)o; + pyFuncOverload->ensureInitialized(); + return translateExceptionToPyObject([&] { if (!PyUnicode_Check(attrName)) { throw std::runtime_error("Expected a string for attribute name"); diff --git a/typed_python/PyFunctionOverload.hpp b/typed_python/PyFunctionOverload.hpp index 94bf6f19b..f6ba23830 100644 --- a/typed_python/PyFunctionOverload.hpp +++ b/typed_python/PyFunctionOverload.hpp @@ -28,10 +28,14 @@ class PyFunctionOverload { PyObject* mDict; + bool mIsInitialized; + FunctionOverload& getOverload(); void initFields(); + void ensureInitialized(); + static PyObject* tp_getattro(PyObject *o, PyObject* attrName); static PyObject* tp_repr(PyObject *self); diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 4fd208453..3601f7514 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1046,6 +1046,8 @@ MutuallyRecursiveTypeGroup* PythonSerializationContext::deserializeMutuallyRecur }); } else if (fieldNumber == 4) { + b.consumeCompoundMessage(wireType, [&](size_t indexInGroup, size_t subWireType) {}); + // now that we've constructed the type diagram, walk over all the types // and finalize them. We have to do this before we wire function globals // because sometime the function globals hold python references to type objects diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index 954c9ae3c..b059e7fd8 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -811,6 +811,9 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec b.writeEndCompound(); + b.writeBeginCompound(4); + b.writeEndCompound(); + b.writeEndCompound(); } diff --git a/typed_python/compiler/python_to_native_converter.py b/typed_python/compiler/python_to_native_converter.py index 796ed4ac6..e385f1503 100644 --- a/typed_python/compiler/python_to_native_converter.py +++ b/typed_python/compiler/python_to_native_converter.py @@ -594,8 +594,8 @@ def convertTypedFunctionCall(self, functionType, overloadIx, inputWrappers, asse overload.name, overload.functionCode, overload.realizedGlobals, - overload.functionGlobals, - overload.funcGlobalsInCells, + overload.globals, + {}, list(overload.closureVarLookups), realizedInputWrappers, returnType, From 3fcadfe35b6804628fad0f5bb33463cc5305ce97 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sun, 16 Jul 2023 00:59:49 +0000 Subject: [PATCH 47/83] Fix some serialization bugs. --- repro.py | 13 +++--- typed_python/CompilerVisiblePyObj.hpp | 62 +++++++++++++++++---------- typed_python/FunctionGlobal.hpp | 4 +- 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/repro.py b/repro.py index fae9c7b12..0b22deb47 100644 --- a/repro.py +++ b/repro.py @@ -1,14 +1,17 @@ import sys -from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final, Function, ListOf +from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final, Function, ListOf, NotCompiled from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr + def writer(): - class Base(Class): - def f(self): - return self + @NotCompiled + def fn(x) -> str: + return str(x) + + print(fn.overloads[0].globals) - bytesToWrite = SerializationContext().serialize(Base) + bytesToWrite = SerializationContext().serialize(fn) with open("a.dat", "wb") as f: f.write(bytesToWrite) diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 2950a1729..07a3ba719 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -179,30 +179,39 @@ class CompilerVisiblePyObj { template void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { + uint32_t id; + bool isNew; + std::tie(id, isNew) = buffer.cachePointer((void*)this, nullptr); - TODO: implement memoize! - - buffer.writeBeginCompound(fieldNumber); - buffer.writeRegisterType(0, (uint64_t)mKind); - - if (mKind == Kind::Type) { - context.serializeNativeType(mType, buffer, 1); - } else - if (mKind == Kind::Instance) { - buffer.writeBeginCompound(2); - context.serializeNativeType(mInstance.type(), buffer, 0); - mInstance.type()->serialize(mInstance.data(), buffer, 1); + if (!isNew) { + buffer.writeBeginCompound(fieldNumber); + buffer.writeUnsignedVarintObject(0, id); buffer.writeEndCompound(); - } else - if (mKind == Kind::PyTuple) { - buffer.writeBeginCompound(3); - for (long i = 0; i < mElements.size(); i++) { - mElements[i]->serialize(context, buffer, i); + return; + } else { + buffer.writeBeginCompound(fieldNumber); + buffer.writeUnsignedVarintObject(0, id); + buffer.writeUnsignedVarintObject(1, (int)mKind); + + if (mKind == Kind::Type) { + context.serializeNativeType(mType, buffer, 2); + } else + if (mKind == Kind::Instance) { + buffer.writeBeginCompound(3); + context.serializeNativeType(mInstance.type(), buffer, 0); + mInstance.type()->serialize(mInstance.data(), buffer, 1); + buffer.writeEndCompound(); + } else + if (mKind == Kind::PyTuple) { + buffer.writeBeginCompound(4); + for (long i = 0; i < mElements.size(); i++) { + mElements[i]->serialize(context, buffer, i); + } + buffer.writeEndCompound(); } + buffer.writeEndCompound(); } - - buffer.writeEndCompound(); } template @@ -213,24 +222,33 @@ class CompilerVisiblePyObj { std::vector vec; ::Instance i; + uint32_t id = -1; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { - buffer.readRegisterType(&kind, wireType); + id = buffer.readUnsignedVarint(); } else if (fieldNumber == 1) { - type = context.deserializeNativeType(buffer, wireType); + kind = buffer.readUnsignedVarint(); } else if (fieldNumber == 2) { - i = context.deserializeNativeInstance(buffer, wireType); + type = context.deserializeNativeType(buffer, wireType); } else if (fieldNumber == 3) { + i = context.deserializeNativeInstance(buffer, wireType); + } else + if (fieldNumber == 4) { buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { vec.push_back(CompilerVisiblePyObj::deserialize(context, buffer, wireType)); }); } }); + void* ptr = buffer.lookupCachedPointer(id); + if (ptr) { + return (CompilerVisiblePyObj*)ptr; + } + if (kind == (int)Kind::Type) { if (!type) { throw std::runtime_error("Corrupt CompilerVisiblePyObj::Type"); diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index a304c2a93..6cc79a574 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -39,7 +39,7 @@ class FunctionGlobal { mKind(inKind), mModuleDictOrCell(incref(dictOrCell)), mName(name), - mModuleName(name), + mModuleName(moduleName), mConstant(constant) { } @@ -449,7 +449,7 @@ class FunctionGlobal { return FunctionGlobal::NamedModuleMember(cellOrModule, name, moduleName); } if (kind == (int)GlobalType::Constant) { - if (constant) { + if (!constant) { throw std::runtime_error("Corrupt FunctionGlobal - invalid constant"); } return FunctionGlobal::Constant(constant); From 6b4362ea1600c2b9c4eb8c9bc0b9248d5ca40ed0 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sun, 16 Jul 2023 11:41:42 -0400 Subject: [PATCH 48/83] Fix some bugs --- typed_python/FunctionGlobal.hpp | 1 - typed_python/FunctionType.hpp | 17 ++++++++++++++-- typed_python/internals.py | 9 +++++++-- typed_python/type_construction_test.py | 27 ++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 6cc79a574..9a9d230bb 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -467,7 +467,6 @@ class FunctionGlobal { return FunctionGlobal::GlobalInCell((PyObject*)cellOrModule); } - asm("int3"); throw std::runtime_error("Corrupt FunctionGlobal - invalid 'kind'"); } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index ed6f7f318..2769f629c 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -439,11 +439,24 @@ class Function : public Type { } Function* withEntrypoint(bool isEntrypoint) const { - return Function::Make(mRootName, mQualname, mModulename, mOverloads, mClosureType, isEntrypoint, mIsNocompile); + Function* f = Function::Make( + mRootName, mQualname, mModulename, mOverloads, mClosureType, isEntrypoint, mIsNocompile + ); + + if (f->isForwardDefined() && !isForwardDefined()) { + return (Function*)f->forwardResolvesTo(); + } + return f; } Function* withNocompile(bool isNocompile) const { - return Function::Make(mRootName, mQualname, mModulename, mOverloads, mClosureType, mIsEntrypoint, isNocompile); + Function* f = Function::Make( + mRootName, mQualname, mModulename, mOverloads, mClosureType, mIsEntrypoint, isNocompile + ); + if (f->isForwardDefined() && !isForwardDefined()) { + return (Function*)f->forwardResolvesTo(); + } + return f; } Type* getClosureType() const { diff --git a/typed_python/internals.py b/typed_python/internals.py index 9a9caa1a1..c67406633 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -382,8 +382,13 @@ def __instancecheck__(cls, instance): def Function(f, returnTypeOverride=None, assumeClosuresGlobal=False): """Turn a normal python function into a 'typed_python.Function' which obeys type restrictions.""" - return makeFunctionType( - f.__name__, f, assumeClosuresGlobal=assumeClosuresGlobal, returnTypeOverride=returnTypeOverride + return typed_python._types.resolveForwardDefinedType( + makeFunctionType( + f.__name__, + f, + assumeClosuresGlobal=assumeClosuresGlobal, + returnTypeOverride=returnTypeOverride + ) )(f) diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 40cbac1c6..93afda161 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -32,6 +32,33 @@ def g(): assert m['h1'] == m['h2'] +def test_function_global_resolving_to_builtin(): + @NotCompiled + def f(): + return str(x) + + assert f.overloads[0].globals['str'].getValue() is str + + +def test_reference_to_builtin_is_resolvable(): + A = Alternative("A", X={}, f=lambda self: str(self) + B) + + # A is forward defined because it depends on B + assert isForwardDefined(A) + + assert A.f.overloads[0].globals['str'].kind == 'NamedModuleMember' + assert A.f.overloads[0].globals['str'].moduleName == 'builtins' + assert A.f.overloads[0].globals['str'].name == 'str' + + +def test_reference_to_builtin_doesnt_prevent_autoresolve(): + A = Alternative("A", X={}, f=lambda self: str(self)) + + assert not isForwardDefined(A) + + assert A().f() == str(A()) + + def test_type_looks_resolvable_alternative(): A = Alternative("A_", X={}, f=lambda self: B) From 33cefc6f24fa0dcc27f867d414037be6834dd78b Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 17 Jul 2023 03:22:49 +0000 Subject: [PATCH 49/83] CVPO's now know how to internalize themselves. --- typed_python/CompilerVisibleObjectVisitor.hpp | 239 +++++++++--------- typed_python/CompilerVisiblePyObj.hpp | 216 +++++++++++++++- typed_python/FunctionGlobal.hpp | 38 ++- typed_python/FunctionOverload.hpp | 12 + typed_python/FunctionType.hpp | 11 + typed_python/PyCompilerVisiblePyObj.cpp | 6 +- typed_python/Type.cpp | 12 + typed_python/compiler/runtime.py | 2 +- typed_python/type_construction_test.py | 13 + typed_python/types_serialization_test.py | 4 + 10 files changed, 422 insertions(+), 131 deletions(-) diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index 30f21fd55..d40e9fb0d 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -173,6 +173,119 @@ class LambdaVisitor { mTopoVisitor(topo); } + void visitInstance( + Type* objType, + instance_ptr instance + ) const { + visitTopo(TypeOrPyobj(objType)); + + if (objType->isComposite()) { + CompositeType* compType = (CompositeType*)objType; + for (long k = 0; k < compType->getTypes().size(); k++) { + visitInstance( + compType->getTypes()[k], + compType->eltPtr(instance, k) + ); + } + return; + } + + if (objType->isFunction()) { + // visit the function's closure + Function* funcT = (Function*)objType; + visitInstance( + funcT->getClosureType(), + instance + ); + return; + } + + if (objType->isOneOf()) { + OneOfType* o = (OneOfType*)objType; + auto typeAndData = o->unwrap(instance); + visitInstance(typeAndData.first, typeAndData.second); + return; + } + + if (objType->isAlternative()) { + Alternative* a = (Alternative*)objType; + visitHash(ShaHash(a->which(instance))); + auto typeAndData = a->unwrap(instance); + visitInstance(typeAndData.first, typeAndData.second); + return; + } + + if (objType->isConcreteAlternative()) { + ConcreteAlternative* a = (ConcreteAlternative*)objType; + visitInstance(a->getAlternative(), instance); + return; + } + + if (objType->isTupleOf()) { + TupleOfType* a = (TupleOfType*)objType; + size_t count = a->count(instance); + + visitHash(ShaHash(count)); + for (long k = 0; k < count; k++) { + visitInstance(a->getEltType(), a->eltPtr(instance, k)); + } + + return; + } + + if (objType->isConstDict()) { + ConstDictType* a = (ConstDictType*)objType; + size_t count = a->count(instance); + + visitHash(ShaHash(count)); + for (long k = 0; k < count; k++) { + visitInstance(a->keyType(), a->kvPairPtrKey(instance, k)); + visitInstance(a->valueType(), a->kvPairPtrValue(instance, k)); + } + + return; + } + + if (objType->isPyCell()) { + static PyCellType* pct = PyCellType::Make(); + + PyObject* o = pct->getPyObj(instance); + + if (!PyCell_Check(o) || !PyCell_Get(o)) { + return; + } + + visitTopo(PyCell_Get(o)); + return; + } + + if (objType->isBool()) { + visitHash(ShaHash(*(bool*)instance ? 1 : 0)); + return; + } + + if (objType->isRegister()) { + visitHash(ShaHash::SHA1((void*)instance, objType->bytecount())); + return; + } + if (objType->isString()) { + visitHash(ShaHash(((StringType*)objType)->toUtf8String(instance))); + return; + } + if (objType->isBytes()) { + size_t ct = ((BytesType*)objType)->count(instance); + visitHash(ShaHash(ct)); + + if (ct) { + visitHash(ShaHash::SHA1( + ((BytesType*)objType)->eltPtr(instance, 0), + ct + )); + } + return; + } + } + void visitNamedTopo(std::string name, TypeOrPyobj instance) const { mNamedVisitor(name, instance); } @@ -570,124 +683,6 @@ class CompilerVisibleObjectVisitor { ); } - template - static void walkInstance( - Type* objType, - instance_ptr instance, - const visitor_type& visitor - ) { - visitor.visitTopo(TypeOrPyobj(objType)); - - if (objType->isComposite()) { - CompositeType* compType = (CompositeType*)objType; - for (long k = 0; k < compType->getTypes().size(); k++) { - walkInstance( - compType->getTypes()[k], - compType->eltPtr(instance, k), - visitor - ); - } - return; - } - - if (objType->isFunction()) { - // visit the function's closure - Function* funcT = (Function*)objType; - walkInstance( - funcT->getClosureType(), - instance, - visitor - ); - return; - } - - if (objType->isOneOf()) { - OneOfType* o = (OneOfType*)objType; - auto typeAndData = o->unwrap(instance); - walkInstance(typeAndData.first, typeAndData.second, visitor); - return; - } - - if (objType->isAlternative()) { - Alternative* a = (Alternative*)objType; - visitor.visitHash(ShaHash(a->which(instance))); - auto typeAndData = a->unwrap(instance); - walkInstance(typeAndData.first, typeAndData.second, visitor); - return; - } - - if (objType->isConcreteAlternative()) { - ConcreteAlternative* a = (ConcreteAlternative*)objType; - walkInstance(a->getAlternative(), instance, visitor); - return; - } - - if (objType->isTupleOf()) { - TupleOfType* a = (TupleOfType*)objType; - size_t count = a->count(instance); - - visitor.visitHash(ShaHash(count)); - for (long k = 0; k < count; k++) { - walkInstance(a->getEltType(), a->eltPtr(instance, k), visitor); - } - - return; - } - - if (objType->isConstDict()) { - ConstDictType* a = (ConstDictType*)objType; - size_t count = a->count(instance); - - visitor.visitHash(ShaHash(count)); - for (long k = 0; k < count; k++) { - walkInstance(a->keyType(), a->kvPairPtrKey(instance, k), visitor); - walkInstance(a->valueType(), a->kvPairPtrValue(instance, k), visitor); - } - - return; - } - - if (objType->isPyCell()) { - static PyCellType* pct = PyCellType::Make(); - - PyObject* o = pct->getPyObj(instance); - - if (!PyCell_Check(o) || !PyCell_Get(o)) { - return; - } - - visitor.visitTopo(PyCell_Get(o)); - return; - } - - if (objType->isBool()) { - visitor.visitHash(ShaHash(*(bool*)instance ? 1 : 0)); - return; - } - - if (objType->isRegister()) { - visitor.visitHash(ShaHash::SHA1((void*)instance, objType->bytecount())); - return; - } - if (objType->isString()) { - visitor.visitHash(ShaHash(((StringType*)objType)->toUtf8String(instance))); - return; - } - if (objType->isBytes()) { - size_t ct = ((BytesType*)objType)->count(instance); - visitor.visitHash(ShaHash(ct)); - - if (ct) { - visitor.visitHash(ShaHash::SHA1( - ((BytesType*)objType)->eltPtr(instance, 0), - ct - )); - } - return; - } - } - - template static void walk( TypeOrPyobj obj, @@ -743,11 +738,9 @@ class CompilerVisibleObjectVisitor { if (argType) { visitor.visitHash(ShaHash(2)); visitor.visitTopo(argType); - - walkInstance( - argType, - ((PyInstance*)obj.pyobj())->dataPtr(), - visitor + visitor.visitInstance( + argType, + ((PyInstance*)obj.pyobj())->dataPtr() ); return; } diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 07a3ba719..2fde27587 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -32,15 +32,27 @@ and singletonish class CompilerVisiblePyObj { enum class Kind { + // this should never be visible in a running program Uninitialized = 0, + // we're pointing back into a fully resolved Type Type = 1, + // we're pointing into a TP instannce that doesn't reach + // a more complex object. It can have Type leaves in it Instance = 2, + // this is a python tuple object PyTuple = 3, + // the bailout pathway for cases we don't handle well. We should + // assume that the compiler will treat this object as a plain + // PyObject without looking inside of it, and the details of this + // object are insufficient to differentiate two different Function + // types that both refer to different ArbitraryPyObject instances. + ArbitraryPyObject = 4 }; CompilerVisiblePyObj() : mKind(Kind::Uninitialized), - mType(nullptr) + mType(nullptr), + mPyObject(nullptr) { } @@ -61,6 +73,10 @@ class CompilerVisiblePyObj { return mKind == Kind::PyTuple; } + bool isArbitraryPyObject() const { + return mKind == Kind::ArbitraryPyObject; + } + static CompilerVisiblePyObj* Type(Type* t) { CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); @@ -87,6 +103,15 @@ class CompilerVisiblePyObj { return res; } + static CompilerVisiblePyObj* ArbitraryPyObject(PyObject* val) { + CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + + res->mKind = Kind::ArbitraryPyObject; + res->mPyObject = incref(val); + + return res; + } + static CompilerVisiblePyObj* PyTuple(const std::vector& elts) { CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); @@ -96,6 +121,153 @@ class CompilerVisiblePyObj { return res; } + // return a CVPO for 'val', stashing it in 'constantMapCache' + // in case we hit a recursion. + static CompilerVisiblePyObj* internalizePyObj( + PyObject* val, + std::unordered_map& constantMapCache, + const std::map<::Type*, ::Type*>& groupMap + ) { + auto it = constantMapCache.find(val); + + if (it != constantMapCache.end()) { + return it->second; + } + + constantMapCache[val] = new CompilerVisiblePyObj(); + + constantMapCache[val]->becomeInternalizedOf(val, constantMapCache, groupMap); + + return constantMapCache[val]; + } + + void becomeInternalizedOf( + PyObject* val, + std::unordered_map& constantMapCache, + const std::map<::Type*, ::Type*>& groupMap + ) { + ::Type* t = PyInstance::extractTypeFrom(val); + + if (t) { + if (groupMap.find(t) != groupMap.end()) { + t = groupMap.find(t)->second; + } else { + if (t->isForwardDefined()) { + if (t->isResolved()) { + t = t->forwardResolvesTo(); + } + } + } + + mKind = Kind::Type; + mType = t; + return; + } + + if (PyTuple_Check(val)) { + mKind = Kind::PyTuple; + for (long i = 0; i < PyTuple_Size(val); i++) { + mElements.push_back( + CompilerVisiblePyObj::internalizePyObj( + PyTuple_GetItem(val, i), + constantMapCache, + groupMap + ) + ); + } + return; + } + + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); + if (instanceType) { + mKind = Kind::Instance; + mInstance = ::Instance::create( + instanceType, + ((PyInstance*)val)->dataPtr() + ); + return; + } + + if (val == Py_None) { + mKind = Kind::Instance; + return; + } + + if (PyBool_Check(val)) { + mKind = Kind::Instance; + mInstance = Instance::create(val == Py_True); + return; + } + + if (PyLong_Check(val)) { + mKind = Kind::Instance; + + try { + mInstance = Instance::create((int64_t)PyLong_AsLongLong(val)); + } + catch(...) { + mInstance = Instance::create((uint64_t)PyLong_AsUnsignedLongLong(val)); + } + + return; + } + + if (PyFloat_Check(val)) { + mKind = Kind::Instance; + mInstance = Instance::create(PyFloat_AsDouble(val)); + return; + } + + if (PyBytes_Check(val)) { + mKind = Kind::Instance; + mInstance = Instance::createAndInitialize( + BytesType::Make(), + [&](instance_ptr i) { + BytesType::Make()->constructor( + i, + PyBytes_GET_SIZE(val), + PyBytes_AsString(val) + ); + } + ); + return; + } + + if (PyUnicode_Check(val)) { + mKind = Kind::Instance; + + auto kind = PyUnicode_KIND(val); + assert( + kind == PyUnicode_1BYTE_KIND || + kind == PyUnicode_2BYTE_KIND || + kind == PyUnicode_4BYTE_KIND + ); + int64_t bytesPerCodepoint = + kind == PyUnicode_1BYTE_KIND ? 1 : + kind == PyUnicode_2BYTE_KIND ? 2 : + 4 ; + + int64_t count = PyUnicode_GET_LENGTH(val); + + const char* data = + kind == PyUnicode_1BYTE_KIND ? (char*)PyUnicode_1BYTE_DATA(val) : + kind == PyUnicode_2BYTE_KIND ? (char*)PyUnicode_2BYTE_DATA(val) : + (char*)PyUnicode_4BYTE_DATA(val); + + mInstance = Instance::createAndInitialize( + StringType::Make(), + [&](instance_ptr i) { + StringType::Make()->constructor(i, bytesPerCodepoint, count, data); + } + ); + + return; + } + + mKind = Kind::ArbitraryPyObject; + mPyObject = incref(val); + } + void append(CompilerVisiblePyObj* elt) { if (mKind != Kind::PyTuple) { throw std::runtime_error("Expected a PyTuple"); @@ -140,12 +312,16 @@ class CompilerVisiblePyObj { if (mKind == Kind::Instance) { // TODO: what to do here? - throw std::runtime_error("TODO: CompilerVisiblePyObj::_visitCompilerVisibleInternals"); + visitor.visitInstance(mInstance.type(), mInstance.data()); } if (mKind == Kind::PyTuple) { // TODO: what to do here? - throw std::runtime_error("TODO: CompilerVisiblePyObj::_visitCompilerVisibleInternals"); + throw std::runtime_error("TODO: CompilerVisiblePyObj::_visitCompilerVisibleInternals PyTuple"); + } + + if (mKind == Kind::ArbitraryPyObject) { + visitor.visitTopo(mPyObject); } } @@ -156,6 +332,18 @@ class CompilerVisiblePyObj { return (PyObject*)PyInstance::typeObj(mType); } + if (mKind == Kind::ArbitraryPyObject) { + return mPyObject; + } + + if (mKind == Kind::Instance) { + if (!mPyObject) { + mPyObject = PyInstance::extractPythonObject(mInstance); + } + + return mPyObject; + } + throw std::runtime_error("Can't make a python object representation for this pyobj"); } @@ -172,11 +360,14 @@ class CompilerVisiblePyObj { return "CompilerVisiblePyObj.PyTuple()"; } + if (mKind == Kind::ArbitraryPyObject) { + return std::string("CompilerVisiblePyObj.ArbitraryPyObject(type=") + + mPyObject->ob_type->tp_name + ")"; + } + throw std::runtime_error("Unknown CompilerVisiblePyObj Kind."); } - - template void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { uint32_t id; @@ -208,6 +399,9 @@ class CompilerVisiblePyObj { mElements[i]->serialize(context, buffer, i); } buffer.writeEndCompound(); + } else + if (mKind == Kind::ArbitraryPyObject) { + context.serializePythonObject(mPyObject, buffer, 5); } buffer.writeEndCompound(); @@ -223,6 +417,7 @@ class CompilerVisiblePyObj { std::vector vec; ::Instance i; uint32_t id = -1; + PyObjectHolder pyobj; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { @@ -241,6 +436,9 @@ class CompilerVisiblePyObj { buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { vec.push_back(CompilerVisiblePyObj::deserialize(context, buffer, wireType)); }); + } else + if (fieldNumber == 5) { + pyobj.steal(context.deserializePythonObject(buffer, wireType)); } }); @@ -268,6 +466,10 @@ class CompilerVisiblePyObj { return new CompilerVisiblePyObj(); } + if (kind == (int)Kind::ArbitraryPyObject) { + return CompilerVisiblePyObj::ArbitraryPyObject(pyobj); + } + throw std::runtime_error("Corrupt CompilerVisiblePyObj - invalid kind"); } @@ -277,5 +479,9 @@ class CompilerVisiblePyObj { ::Type* mType; ::Instance mInstance; + // if we are an ArbitraryPythonObject this is always populated + // otherwise, it will be a cache for a constructed instance + PyObject* mPyObject; + std::vector mElements; }; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 9a9d230bb..2d21cdbc4 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -292,7 +292,6 @@ class FunctionGlobal { visitor.visitTopo( t->forwardResolvesTo() ); - } else { } } else if (t) { visitor.visitTopo(t); @@ -301,6 +300,43 @@ class FunctionGlobal { } } + FunctionGlobal withConstantsInternalized( + std::unordered_map& constantMapCache, + const std::map& groupMap + ) { + if (!(isGlobalInDict() || isGlobalInCell())) { + return *this; + } + + Type* t = getValueAsType(); + if (t) { + auto it = groupMap.find(t); + if (it != groupMap.end()) { + return FunctionGlobal::Constant( + CompilerVisiblePyObj::Type(it->second) + ); + } else { + return FunctionGlobal::Constant( + CompilerVisiblePyObj::Type(t) + ); + } + } + + PyObject* value = getValueAsPyobj(); + + if (!value) { + return FunctionGlobal::Unbound(); + } + + return FunctionGlobal::Constant( + CompilerVisiblePyObj::internalizePyObj( + value, + constantMapCache, + groupMap + ) + ); + } + FunctionGlobal withUpdatedInternalTypePointers(const std::map& groupMap) { Type* t = getValueAsType(); diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index f426c45e3..d38659664 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -267,6 +267,18 @@ class FunctionOverload { } } + void internalizeConstants( + std::unordered_map& constantMapCache, + const std::map& groupMap + ) { + for (auto& nameAndGlobal: mGlobals) { + nameAndGlobal.second = nameAndGlobal.second.withConstantsInternalized( + constantMapCache, + groupMap + ); + } + } + void updateInternalTypePointers(const std::map& groupMap) { if (mReturnType) { Type::updateTypeRefFromGroupMap(mReturnType, groupMap); diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 2769f629c..59a869ae6 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -158,6 +158,17 @@ class Function : public Type { return new Function(); } + // replace any direct references to PyObject we're holding internally + // with CompilerVisiblePyObj references instead + void internalizeConstants( + std::unordered_map& constantMapCache, + const std::map& groupMap + ) { + for (auto& o: mOverloads) { + o.internalizeConstants(constantMapCache, groupMap); + } + } + void updateInternalTypePointersConcrete(const std::map& groupMap) { updateTypeRefFromGroupMap(mClosureType, groupMap); diff --git a/typed_python/PyCompilerVisiblePyObj.cpp b/typed_python/PyCompilerVisiblePyObj.cpp index de8131fa1..aeb62c819 100644 --- a/typed_python/PyCompilerVisiblePyObj.cpp +++ b/typed_python/PyCompilerVisiblePyObj.cpp @@ -87,7 +87,11 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN } if (obj->isPyTuple()) { - return PyUnicode_FromString("Py2Tuple"); + return PyUnicode_FromString("PyTuple"); + } + + if (obj->isArbitraryPyObject()) { + return PyUnicode_FromString("ArbitraryPyObject"); } throw std::runtime_error("Unknown CompilerVisiblePyObj Kind"); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index ae7837aac..a80c5cf29 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -474,6 +474,18 @@ void Type::attemptToResolve() { typeAndSource.first->updateInternalTypePointers(resolutionMapping); } + // allow each Function type to build CompilerVisiblePythonObjects for its globals + // after this, any FunctionGlobals will refer to Constants not cells + std::unordered_map compilerVisiblePyObjMap; + for (auto typeAndSource: resolutionSource) { + if (typeAndSource.first->isFunction()) { + ((Function*)typeAndSource.first)->internalizeConstants( + compilerVisiblePyObjMap, + resolutionMapping + ); + } + } + // cause each type to recompute its name. We have to do this in a single pass // without any types being aware of their final name before we run the post initialize // step. Otherwise, it's possible for the names to depend on the order of initialization diff --git a/typed_python/compiler/runtime.py b/typed_python/compiler/runtime.py index 314b3e8c0..4c4d4bcdd 100644 --- a/typed_python/compiler/runtime.py +++ b/typed_python/compiler/runtime.py @@ -351,7 +351,7 @@ def compileFunctionOverload(self, functionType, overloadIx, arguments, arguments overloadIx, fp.fp, callTarget.output_type.typeRepresentation if callTarget.output_type is not None else type(None), - [i.typeRepresentation for i in callTarget.input_types] + tuple([i.typeRepresentation for i in callTarget.input_types]) ) self.converter.flushDelayedVMIs() diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 93afda161..153cf2fa4 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -51,6 +51,19 @@ def test_reference_to_builtin_is_resolvable(): assert A.f.overloads[0].globals['str'].name == 'str' +def test_function_cell_bindings_are_resolved_to_constants(): + def CofX(x): + class C(Class): + def f(self): + return x + + return C + + assert CofX(1).f.overloads[0].globals['x'].kind == 'Constant' + assert CofX(1).f.overloads[0].globals['x'].getValue() is 1 + assert CofX(2).f.overloads[0].globals['x'].getValue() is 2 + + def test_reference_to_builtin_doesnt_prevent_autoresolve(): A = Alternative("A", X={}, f=lambda self: str(self)) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 205e13040..e4cbd5d3f 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2518,6 +2518,10 @@ def f(self): return B + B1 = deserializeAndCall(1) + print(B1.f.overloads[0].globals['x']) + return; + assert callFunctionInFreshProcess(deserializeAndCall, (1,)) is deserializeAndCall(1) assert callFunctionInFreshProcess(deserializeAndCall, (1,)) is not deserializeAndCall(2) assert callFunctionInFreshProcess(deserializeAndCall, (2,)) is deserializeAndCall(2) From 78b1c80eb1458a19146bf80d9c8e6793e8fd94d1 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 17 Jul 2023 22:50:53 +0000 Subject: [PATCH 50/83] working on 'test_globals_of_entrypointed_functions_serialized_externally' --- repro.py | 1 + typed_python/internals.py | 6 +++--- typed_python/types_serialization_test.py | 10 +++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/repro.py b/repro.py index 0b22deb47..6582553cc 100644 --- a/repro.py +++ b/repro.py @@ -4,6 +4,7 @@ from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr + def writer(): @NotCompiled def fn(x) -> str: diff --git a/typed_python/internals.py b/typed_python/internals.py index c67406633..df2352695 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -384,9 +384,9 @@ def Function(f, returnTypeOverride=None, assumeClosuresGlobal=False): """Turn a normal python function into a 'typed_python.Function' which obeys type restrictions.""" return typed_python._types.resolveForwardDefinedType( makeFunctionType( - f.__name__, - f, - assumeClosuresGlobal=assumeClosuresGlobal, + f.__name__, + f, + assumeClosuresGlobal=assumeClosuresGlobal, returnTypeOverride=returnTypeOverride ) )(f) diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index e4cbd5d3f..d8208e844 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2083,8 +2083,8 @@ def test_names_of_builtin_alternatives(self): assert sc.deserialize(sc.serialize(TupleOf(native_ast.Type))) is TupleOf(native_ast.Type) assert sc.deserialize(sc.serialize(NamedCallTarget)) is NamedCallTarget - assert len(sc.serialize(native_ast.Type)) < 100 - assert len(sc.serialize(TupleOf(native_ast.Type))) < 100 + assert len(sc.serialize(native_ast.Type)) < 200 + assert len(sc.serialize(TupleOf(native_ast.Type))) < 200 def test_badly_named_module_works(self): sc = SerializationContext() @@ -2775,12 +2775,14 @@ def makeC(): path = os.path.join(tempdir, "asdf.py") CONTENTS = ( - "from typed_python import Entrypoint, ListOf, Class, Final\n" + "from typed_python import Entrypoint, ListOf, Class, Final, isForwardDefined, Member\n" "class C(Class, Final):\n" " @staticmethod\n" " @Entrypoint\n" " def anF():\n" " return C\n" + "print(isForwardDefined(C))\n" + "print(C.anF.overloads[0].globals)\n" ) with open(path, "w") as f: @@ -2796,6 +2798,8 @@ def makeC(): s = SerializationContext() return s.serialize(globals['C']) + makeC() + return serializedC = callFunctionInFreshProcess(makeC, ()) C = SerializationContext().deserialize(serializedC) From 36b00aa9a284339ee31c98f0dc3ddfde86ac9e47 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 18 Jul 2023 03:48:19 +0000 Subject: [PATCH 51/83] Expose source of original forward definitions and ensure that Classes expose members. The problem we're having is that 'Function' and 'Entrypoint' force resolution of their Function objects immediately. In this case, the forward function refers to a Global whose module dict doesn't have a name and is not yet defined. As a result, we end up forcing the resolution of the global immediately which forces it to 'Unbound'. This is kind of a weird test because we're constructing a global inside of a module whose name is not known. How should we handle this? This is similar to what we do in ModuleRepresentation. At some level, the problem is that we are conflating function types with function instances, which happens when we mark them with @Entrypoint inside of a class definition. When we @Function or @Entrypoint we are forcing the objects to resolve so we can instantiate them. But this is different in a Class object where we are intending for the function to be part of a Class. Of course, in a ModuleRepresentation, we would still want to acknowledge that the Function is unresolved. Ultimately the rule should be: (1) when you @Function/@Entrypoint a function object, you get something you can use. * the 'future uncertainty' embedded in the object is not part of its type. * when you pass it into 'the compiler' we force it to resolve or be unresolved but managed by the system --- typed_python/FunctionOverload.hpp | 2 + typed_python/PyClassInstance.cpp | 160 ++++++++++++++--------- typed_python/Type.cpp | 6 + typed_python/Type.hpp | 19 +++ typed_python/__init__.py | 3 +- typed_python/_types.cpp | 39 ++++-- typed_python/type_construction_test.py | 49 ++++++- typed_python/types_serialization_test.py | 4 +- 8 files changed, 209 insertions(+), 73 deletions(-) diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index d38659664..bae6d146f 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -772,6 +772,8 @@ class FunctionOverload { ); outGlobals[ref.first] = ref.second; + + std::cout << "have " << ref.first << " = " << ref.second.toString() << "\n"; } } diff --git a/typed_python/PyClassInstance.cpp b/typed_python/PyClassInstance.cpp index 9f08cb2b5..a8e9a1287 100644 --- a/typed_python/PyClassInstance.cpp +++ b/typed_python/PyClassInstance.cpp @@ -658,43 +658,64 @@ PyObject* PyClassInstance::tpGetattrGeneric( return ret; } -/* initialize the type object's dict. -if 'asHeldClass', then this is the held class's dict we're initializing. -*/ -void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyTypeObject* pyType, bool asHeldClass) { - PyObjectStealer bases(PyTuple_New(classT->getHeldClass()->getBases().size())); +PyObject* convertPyTupleEltToPyObj(std::string s) { + return PyUnicode_FromString(s.c_str()); +} - for (long k = 0; k < classT->getHeldClass()->getBases().size(); k++) { - PyTuple_SetItem( - bases, - k, - incref(typePtrToPyTypeRepresentation(classT->getHeldClass()->getBases()[k]->getClassType())) - ); - } +PyObject* convertPyTupleEltToPyObj(PyObject* o) { + return o; +} - PyObjectStealer mro(PyTuple_New(classT->getHeldClass()->getMro().size())); +PyObject* convertPyTupleEltToPyObj(Type* t) { + return incref(PyInstance::typePtrToPyTypeRepresentation(t)); +} - for (long k = 0; k < classT->getHeldClass()->getMro().size(); k++) { - PyTuple_SetItem( - mro, - k, - incref(typePtrToPyTypeRepresentation(classT->getHeldClass()->getMro()[k]->getClassType())) - ); +template +PyObject* newPyTuple(const container_type& container, const func_type& objCreator) { + PyObject* tup = PyTuple_New(container.size()); + long ix = 0; + for (auto containerElt: container) { + PyTuple_SetItem(tup, ix, convertPyTupleEltToPyObj(objCreator(containerElt))); } + return tup; +} + +/* initialize the type object's dict. - PyObjectStealer types(PyTuple_New(classT->getMembers().size())); +if 'asHeldClass', then this is the held class's dict we're initializing. +*/ +void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyTypeObject* pyType, bool asHeldClass) { + PyObjectStealer bases( + newPyTuple( + classT->getHeldClass()->getBases(), + [&](HeldClass* t) { return t->getClassType(); } + ) + ); - for (long k = 0; k < classT->getMembers().size(); k++) { - PyTuple_SetItem(types, k, incref(typePtrToPyTypeRepresentation(classT->getMembers()[k].getType()))); - } + PyObjectStealer mro( + newPyTuple( + classT->getHeldClass()->getMro(), + [&](HeldClass* t) { + return t->getClassType(); + } + ) + ); - PyObjectStealer names(PyTuple_New(classT->getMembers().size())); - for (long k = 0; k < classT->getMembers().size(); k++) { - PyObject* namePtr = PyUnicode_FromString(classT->getMembers()[k].getName().c_str()); - PyTuple_SetItem(names, k, namePtr); - } + PyObjectStealer memberTypes( + newPyTuple( + classT->isForwardDefined() ? classT->getOwnMembers() : classT->getMembers(), + [&](const MemberDefinition& member) { return member.getType(); } + ) + ); + PyObjectStealer names( + newPyTuple( + classT->isForwardDefined() ? classT->getOwnMembers() : classT->getMembers(), + [&](const MemberDefinition& member) { return member.getName(); } + ) + ); + PyObjectStealer defaults(PyDict_New()); for (long k = 0; k < classT->getMembers().size(); k++) { @@ -710,29 +731,38 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT defaults, classT->getHeldClass()->getMemberName(k).c_str(), defaultVal - ); + ); } } PyObjectStealer memberFunctions(PyDict_New()); - for (auto p: classT->getMemberFunctions()) { + for (auto p: classT->isForwardDefined() ? classT->getOwnMemberFunctions() : classT->getMemberFunctions()) { PyDict_SetItemString(memberFunctions, p.first.c_str(), typePtrToPyTypeRepresentation(p.second)); - // TODO: find a predefined function that does this method search - PyMethodDef* defined = pyType->tp_methods; - while (defined && defined->ml_name && !!strcmp(defined->ml_name, p.first.c_str())) - defined++; - - if (!defined || !defined->ml_name) { - if (p.second->getClosureType()->bytecount()) { - std::cout << "WARNING: invalid class member " << classT->name() << p.first << " had a nonempty closure.\n"; - } else { - if (p.second->bytecount()) { - throw std::runtime_error("Somehow, a Class got a function with a closure type."); + if (!classT->isForwardDefined()) { + // TODO: clean this up. I'm not sure what we are guarding against - is it + // the case that sometimes we overwrite something already in the dict? + PyMethodDef* defined = pyType->tp_methods; + while (defined && defined->ml_name && !!strcmp(defined->ml_name, p.first.c_str())) + defined++; + + if (!defined || !defined->ml_name) { + if (p.second->getClosureType()->bytecount()) { + std::cout << "WARNING: invalid class member " << classT->name() << p.first << " had a nonempty closure.\n"; + } else { + if (p.second->bytecount()) { + throw std::runtime_error("Somehow, a Class got a function with a closure type."); + } + PyDict_SetItemString(pyType->tp_dict, p.first.c_str(), PyInstance::initialize(p.second, [&](instance_ptr) {})); } - PyDict_SetItemString(pyType->tp_dict, p.first.c_str(), PyInstance::initialize(p.second, [&](instance_ptr) {})); } + } else { + PyDict_SetItemString( + pyType->tp_dict, + p.first.c_str(), + typePtrToPyTypeRepresentation(p.second) + ); } } @@ -749,7 +779,7 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT PyDict_SetItemString(pyType->tp_dict, "HeldClass", typePtrToPyTypeRepresentation(classT->getHeldClass())); } - PyDict_SetItemString(pyType->tp_dict, "MemberTypes", types); + PyDict_SetItemString(pyType->tp_dict, "MemberTypes", memberTypes); PyDict_SetItemString(pyType->tp_dict, "BaseClasses", bases); PyDict_SetItemString(pyType->tp_dict, "IsFinal", classT->isFinal() ? Py_True : Py_False); PyDict_SetItemString(pyType->tp_dict, "MRO", mro); @@ -777,25 +807,37 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT PyObjectStealer staticMemberFunctions(PyDict_New()); PyDict_SetItemString(pyType->tp_dict, "StaticMemberFunctions", staticMemberFunctions); - for (auto nameAndObj: classT->getStaticFunctions()) { - PyDict_SetItemString(staticMemberFunctions, nameAndObj.first.c_str(), typePtrToPyTypeRepresentation(nameAndObj.second)); + for (auto nameAndObj: classT->isForwardDefined() ? + classT->getOwnStaticFunctions() : classT->getStaticFunctions()) { + PyDict_SetItemString( + staticMemberFunctions, + nameAndObj.first.c_str(), typePtrToPyTypeRepresentation(nameAndObj.second) + ); - if (nameAndObj.second->getClosureType()->bytecount()) { - throw std::runtime_error( - "Somehow, " + classT->name() + "." - + nameAndObj.first + " has a populated closure." + if (!classT->isForwardDefined()) { + if (nameAndObj.second->getClosureType()->bytecount()) { + throw std::runtime_error( + "Somehow, " + classT->name() + "." + + nameAndObj.first + " has a populated closure." + ); + } + + PyDict_SetItemString( + pyType->tp_dict, + nameAndObj.first.c_str(), + PyStaticMethod_New( + PyInstance::initialize(nameAndObj.second, [&](instance_ptr data){ + //nothing to do - functions like this are just types. + }) + ) + ); + } else { + PyDict_SetItemString( + pyType->tp_dict, + nameAndObj.first.c_str(), + typePtrToPyTypeRepresentation(nameAndObj.second) ); } - - PyDict_SetItemString( - pyType->tp_dict, - nameAndObj.first.c_str(), - PyStaticMethod_New( - PyInstance::initialize(nameAndObj.second, [&](instance_ptr data){ - //nothing to do - functions like this are just types. - }) - ) - ); } PyObjectStealer classMemberFunctions(PyDict_New()); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index a80c5cf29..3fd8902ce 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -583,6 +583,12 @@ void Type::attemptToResolve() { if (!m_forward_resolves_to) { throw std::runtime_error("Somehow, we didn't resolve???"); } + + for (auto typeAndTarget: resolutionMapping) { + if (typeAndTarget.first->isForwardDefined()) { + typeAndTarget.second->addForwardDefinition(typeAndTarget.first); + } + } } void Type::internalize() { diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 63e12ccec..f3fa85eb1 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -927,6 +927,10 @@ class Type { void typeFinishedBeingDeserializedPhase1(); void typeFinishedBeingDeserializedPhase2(); + const std::vector& getForwardDefinitions() const { + return mForwardDefinitions; + } + protected: Type(TypeCategory in_typeCategory) : m_typeCategory(in_typeCategory), @@ -964,6 +968,9 @@ class Type { std::vector mAutoresolveFrameOwners; + // the original forwards that defined this Type + std::vector mForwardDefinitions; + // a sha-hash that uniquely identifies this type. If this value is // the same for two types, then they should be indistinguishable except // for pointers values. @@ -1039,6 +1046,18 @@ class Type { static std::map mInternalizedIdentityHashToType; public: + void addForwardDefinition(Type* t) { + if (!t->isForwardDefined()) { + throw std::runtime_error("Can't add a non-forward defined type as a reverse Forward link"); + } + + if (isForwardDefined()) { + throw std::runtime_error("Can't add a forward definition to a forward type - it should be resolved!"); + } + + mForwardDefinitions.push_back(t); + } + void recomputeName() { TypeStack stack; m_name = computeRecursiveName(stack); diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 6aad85446..822911f0f 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -55,7 +55,8 @@ isForwardDefined, typeLooksResolvable, resolveForwardDefinedType, - identityHash + identityHash, + forwardDefinitionsFor ) import typed_python._types as _types import threading diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 1e6e02eef..6977db68e 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2705,24 +2705,40 @@ PyObject *assertTypeResolvable(PyObject* nullValue, PyObject* args, PyObject* kw }); } -PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { +PyObject *forwardDefinitionsFor(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "resolveForwardDefinedType takes 1 positional argument"); + PyErr_SetString(PyExc_TypeError, "forwardDefinitionsFor takes 1 positional argument"); return NULL; } PyObjectHolder a1(PyTuple_GetItem(args, 0)); - // Type* typeOfArg = PyInstance::extractTypeFrom(((PyObject*)a1)->ob_type, false); + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + + if (!t) { + PyErr_SetString(PyExc_TypeError, "first argument to 'forwardDefinitionsFor' must be a type object"); + return NULL; + } - // if (typeOfArg && typeOfArg->isFunction()) { - // return ::translateExceptionToPyObject([&]() { - // Type* newType = typeOfArg->forwardResolvesTo(); + return ::translateExceptionToPyObject([&]() { + PyObject* res = PyList_New(0); - // return PyInstance::initialize(newType, [&](instance_ptr data) { - // ((Function*)newType)->getClosureType()->copy_constructor(data, ((PyInstance*)(PyObject*)a1)->dataPtr()); - // }); - // }); - // } + for (auto fwd: t->getForwardDefinitions()) { + PyList_Append( + res, + (PyObject*)PyInstance::typeObj(fwd) + ); + } + + return res; + }); +} + +PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { + if (PyTuple_Size(args) != 1) { + PyErr_SetString(PyExc_TypeError, "resolveForwardDefinedType takes 1 positional argument"); + return NULL; + } + PyObjectHolder a1(PyTuple_GetItem(args, 0)); Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); @@ -3669,6 +3685,7 @@ static PyMethodDef module_methods[] = { {"isForwardDefined", (PyCFunction)isForwardDefined, METH_VARARGS, NULL}, {"isResolved", (PyCFunction)isResolved, METH_VARARGS, NULL}, {"_enableTypeAutoresolution", (PyCFunction)enableTypeAutoresolution, METH_VARARGS, NULL}, + {"forwardDefinitionsFor", (PyCFunction)forwardDefinitionsFor, METH_VARARGS, NULL}, {"resolveForwardDefinedType", (PyCFunction)resolveForwardDefinedType, METH_VARARGS, NULL}, {"typeLooksResolvable", (PyCFunction)typeLooksResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, {"assertTypeResolvable", (PyCFunction)assertTypeResolvable, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 153cf2fa4..7c3475bac 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -3,7 +3,8 @@ from typed_python import ( TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, - Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled + Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled, + forwardDefinitionsFor ) from typed_python.test_util import CodeEvaluator @@ -32,6 +33,52 @@ def g(): assert m['h1'] == m['h2'] +def test_can_see_original_class_that_defined_a_forward(): + class C(Class): + x = Member(lambda: B) + + CFwd = (C,) + + assert isForwardDefined(C) + + # this is insufficient to trigger an autoresolve because + # we didn't create a new Type instance. + B = int + + C = resolveForwardDefinedType(C) + + assert not isForwardDefined(C) + + assert forwardDefinitionsFor(C) == [CFwd[0]] + + +def test_forward_class_exposes_functions(): + class C(Class): + x = Member(lambda: B) + + def f(self): + return C + + B = int + + CResolved = resolveForwardDefinedType(C) + + assert C.HeldClass.Class is C + assert CResolved.HeldClass.Class is CResolved + + assert not C.IsFinal + assert not CResolved.IsFinal + + assert C.MRO == (C,) + assert CResolved.MRO == (CResolved,) + + assert CResolved.MemberNames == ('x',) + + assert C.MemberNames == ('x',) + + assert C.f.overloads[0].globals['C'].kind == 'GlobalInCell' + + def test_function_global_resolving_to_builtin(): @NotCompiled def f(): diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index d8208e844..3be8dc45c 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2775,13 +2775,15 @@ def makeC(): path = os.path.join(tempdir, "asdf.py") CONTENTS = ( - "from typed_python import Entrypoint, ListOf, Class, Final, isForwardDefined, Member\n" + "from typed_python import Entrypoint, ListOf, Class, Final, isForwardDefined, Member, forwardDefinitionsFor\n" "class C(Class, Final):\n" " @staticmethod\n" " @Entrypoint\n" " def anF():\n" " return C\n" "print(isForwardDefined(C))\n" + "print(forwardDefinitionsFor(C)[0].anF.overloads[0].globals)\n" + "print(isForwardDefined(forwardDefinitionsFor(C)[0]))\n" "print(C.anF.overloads[0].globals)\n" ) From 3fc3b4c37d44e4e85bfa2649c55f489b2f8c9e71 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 19 Jul 2023 09:14:49 -0400 Subject: [PATCH 52/83] More. --- todo.txt | 115 ++++++++++++++++++++++++++++++ typed_python/FunctionOverload.hpp | 2 - 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 todo.txt diff --git a/todo.txt b/todo.txt new file mode 100644 index 000000000..64a9c6a74 --- /dev/null +++ b/todo.txt @@ -0,0 +1,115 @@ +- ensure classes get a Python reference to their held functions + - all Forward defined types need to expose as much of their internals as possible + - not bytecounts or anything like that + - expose code to allow us to get the _first_ forward that defined a given class +- dotted globals? + - if we do this, how do we resolve the dotted global value? + - this is different for identity hashing than it is for serialization? + - what were we doing in the existing system? This is very squirrely and difficult. +- fix the way we walk things in MRTG - should be unified +- ensure that TypeFunction works as intended now. Some of the weird issues should be resolved now +- fix the ModuleRepresentation stuff +- redocument everything +- finish serialization + - should be able to serialize forward defined types + - ensure we are not leaking CompilerVisiblePyObj values + - really CompilerVisiblePyObj is the wrong name - i think it's more like 'TypeVisiblePyObj' + which reflects the reality that some objects are visible to the typing system and some + are not + - can we get rid of the whole serialize-mrtg thing? at this point, memos should just work right? + - this might leak which would be a problem +- ensure that we don't try to MRTG something that's a forward defined type +- fix the compiler to deal with the new globals thing now that we have it well defined +- figure out how to properly internalize python classes + - the CompilerVisiblePyObj thing needs to actually work + - we need to make sure we can actually destroy these if we're not going to keep them +- get rid of PyObject throughout Function + - should be like a FunctionGlobal + + +def f(x): + class C(Class): + def g(self): + return x + +f(1) vs f(2) + +what kind of global should 'x' be? +- we could leave it a GlobalInCell but it's really not - its a constant and shouldn't be allowed to change + - part of the resolution process needs to get rid of the 'cellness' of this + - fully resolved functions should only have constants, NamedModuleMember instances, and Unbound + + +def f(x): + @Entrypoint + def g(): + return x + return g + +in this case, 'x' is held in the function's closure, so it's not really 'typelike' + + +how should this work: +* we build a graph of unresolved forward types + * these can see each other through cells, module members, and TypeFunction calls that yield Forwards. +* we trigger 'forward resolution' +* we build a complete model of all the python objects and types that are visible to the graph + * named module members are special - we stop the walk when we hit such an object and its not part of our 'resolvability' criteria even if the object is unresolved at that point. + * this is because any references that are through an actual module instead of through explicit + forwards cannot affect our identity, only our compilation + * everything else gets mapped to an explicit 'constant' +* + + +# core ideas: +# 1. it's typelike IFF it can't change during program execution +# 2. module members, class definitions, etc, don't change during program execution once set +# 3. an @Function / @Entrypoint should define an 'untyped' function to whatever degree possible +# since the user isn't providing types +# 4. at the moment we cross Entrypoint boundaries we can look at a function's type graph and +# determine how we want to handle it +# 5. we could have the idea of a 'StatefulClass' holding a class that references some state +# that's independent of its type. This could get passed to its instances +# 6. a Function's default arguments, if stateful, should be part of its data not its type +# 7. if we have an anonymous module, and that module is defined in RF, then that module's identity +# is it's "incoming state" + + +# this is just a regular TP function. nothing funny going on +@Function +def f(): + pass + + +# this should hold 'x' as a NamedModuleMember if this is defined inside of a module +# this is part of the type +@Function +def f(): + return x + + +# if the module is not named, then the module info is not part of the type. +# instead it's part of the Function's closure. 'x' should go through the closure +# and hit the dict indirectly. If the function gets passed to a Class then we'll +# pull all of that apart and re-use it. If it just becomes a free function then +# when we interpret it, it's fine, and when we compile it we get a chance to look carefully +# at it and decide what to do. +__module__ = 'some random module we can't find +@Function +def f(): + return x + + +def makeF(x): + @Function + def f(): + + + + + + + + + + diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index bae6d146f..d38659664 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -772,8 +772,6 @@ class FunctionOverload { ); outGlobals[ref.first] = ref.second; - - std::cout << "have " << ref.first << " = " << ref.second.toString() << "\n"; } } From 581b62500aa74bb3ba9329a15e4f52578655631f Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 19 Jul 2023 21:24:58 -0400 Subject: [PATCH 53/83] [wip] start to work on anonymous globals as state. --- todo.txt | 9 +++++++-- typed_python/PythonObjectOfTypeType.cpp | 5 +++++ typed_python/PythonObjectOfTypeType.hpp | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/todo.txt b/todo.txt index 64a9c6a74..c36b2ecbc 100644 --- a/todo.txt +++ b/todo.txt @@ -5,7 +5,7 @@ - dotted globals? - if we do this, how do we resolve the dotted global value? - this is different for identity hashing than it is for serialization? - - what were we doing in the existing system? This is very squirrely and difficult. + - what were we doing in the existing system? This is very squirrely and difficult. - fix the way we walk things in MRTG - should be unified - ensure that TypeFunction works as intended now. Some of the weird issues should be resolved now - fix the ModuleRepresentation stuff @@ -58,7 +58,7 @@ how should this work: * this is because any references that are through an actual module instead of through explicit forwards cannot affect our identity, only our compilation * everything else gets mapped to an explicit 'constant' -* +* # core ideas: @@ -107,6 +107,11 @@ def makeF(x): +execution plan: +1. if a function's globals dict is not globally visible, then make it part of the _closure_ + * need to allow the function to be recreated correctly given this case +2. allow forward function types to be instantiated if their closures are not forward. This is a forward function instance. +3. allow forward function types to be resolved just like regular function types diff --git a/typed_python/PythonObjectOfTypeType.cpp b/typed_python/PythonObjectOfTypeType.cpp index fb4cacdf5..589a58918 100644 --- a/typed_python/PythonObjectOfTypeType.cpp +++ b/typed_python/PythonObjectOfTypeType.cpp @@ -536,6 +536,11 @@ PythonObjectOfType* PythonObjectOfType::AnyPyObject() { return Make((PyTypeObject*)t); } +PythonObjectOfType* PythonObjectOfType::AnyPyDict() { + static PythonObjectOfType* t = Make((PyTypeObject*)&PyDict_Type); + return t; +} + PythonObjectOfType* PythonObjectOfType::AnyPyType() { PyTypeObject* t = (PyTypeObject*)staticPythonInstance( "typed_python.internals", "type" diff --git a/typed_python/PythonObjectOfTypeType.hpp b/typed_python/PythonObjectOfTypeType.hpp index 92d586ffb..de3602a64 100644 --- a/typed_python/PythonObjectOfTypeType.hpp +++ b/typed_python/PythonObjectOfTypeType.hpp @@ -228,6 +228,8 @@ class PythonObjectOfType : public PyObjectHandleTypeBase { static PythonObjectOfType* AnyPyObject(); + static PythonObjectOfType* AnyPyDict(); + static PythonObjectOfType* AnyPyType(); PyTypeObject* pyType() const { From fa9c71f358c09a378b093a05b2ded4d2121c49b6 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 21 Jul 2023 16:18:20 +0000 Subject: [PATCH 54/83] If we make a typed TP Function object with unnamed globals, we put the globals into the closure just like cells. This lets us not embed an object (namely the globals dict) who doesn't have a formal name into the type of the Function. --- todo.txt | 11 ++- typed_python/ClosureVariableBinding.cpp | 14 ++++ typed_python/FunctionOverload.cpp | 33 ++++++++ typed_python/FunctionOverload.hpp | 37 ++++++++- .../ModuleRepresentationCopyContext.hpp | 1 + typed_python/PyFunctionInstance.cpp | 80 +++++++++++++++---- typed_python/_types.cpp | 46 +++++++++-- typed_python/type_construction_test.py | 23 +++++- typed_python/types_serialization_test.py | 6 -- 9 files changed, 210 insertions(+), 41 deletions(-) diff --git a/todo.txt b/todo.txt index c36b2ecbc..37c15ef12 100644 --- a/todo.txt +++ b/todo.txt @@ -109,12 +109,11 @@ def makeF(x): execution plan: 1. if a function's globals dict is not globally visible, then make it part of the _closure_ + * FunctionGlobal can point to the closure, just like the ClosureVariables + * this is always considered 'resolved' in the sense that such an object is untyped and therefore fully resolved * need to allow the function to be recreated correctly given this case 2. allow forward function types to be instantiated if their closures are not forward. This is a forward function instance. + * forward function instances can't be executed but they can be created. They are placeholder objects ready to be resolved + just like forward types, or to be passed into a Class definition 3. allow forward function types to be resolved just like regular function types - - - - - - + * forward function instances also participate in instance resolution diff --git a/typed_python/ClosureVariableBinding.cpp b/typed_python/ClosureVariableBinding.cpp index 2d1f4dbed..44aa08ad6 100644 --- a/typed_python/ClosureVariableBinding.cpp +++ b/typed_python/ClosureVariableBinding.cpp @@ -9,6 +9,20 @@ Instance ClosureVariableBinding::extractValueOrContainingClosure(Type* closureTy closureType = (Type*)step.getFunction(); } else if (step.isNamedField()) { + if (closureType->isPythonObjectOfType() && closureType == PythonObjectOfType::AnyPyDict()) { + PyObject* dict = PythonObjectOfType::getPyObj(data); + if (!dict || !PyDict_Check(dict)) { + throw std::runtime_error("Invalid closure: expected a populated PyDict"); + } + PyObject* entry = PyDict_GetItemString(dict, step.getNamedField().c_str()); + if (!entry) { + // TODO: this is wrong. Really we need a pattern that lets us communicate + // the fact that this entry is empty and that we should throw a name error. + throw std::runtime_error("Invalid closure: expected " + step.getNamedField() + " of a Dict to be populated."); + } + + return Instance::create(entry); + } else if (closureType->isNamedTuple()) { NamedTuple* tupType = (NamedTuple*)closureType; auto it = tupType->getNameToIndex().find(step.getNamedField()); diff --git a/typed_python/FunctionOverload.cpp b/typed_python/FunctionOverload.cpp index cefb6ceb2..e689742b9 100644 --- a/typed_python/FunctionOverload.cpp +++ b/typed_python/FunctionOverload.cpp @@ -28,6 +28,39 @@ PyObject* FunctionOverload::buildFunctionObj(Type* closureType, instance_ptr clo } } + if (mFunctionGlobalsInClosureVarnames.size()) { + if (mGlobals.size() || globalsInCells.size()) { + throw std::runtime_error( + "Invalid FunctionOverload. We can't convert this to a python function. " + "We are supposed to have our globals either encoded in the globals dict " + "assuming that they are resolvable in a way that can place them in the type " + "or they are supposed to be in our closure. We're not supposed to mix them." + ); + } + + if (!closureType->isTuple() + || ((Tuple*)closureType)->getTypes().size() != 1 + || !((Tuple*)closureType)->getTypes()[0]->isNamedTuple() + ) { + throw std::runtime_error("Invalid closure-enclosing-globals"); + } + + NamedTuple* interior = (NamedTuple*)((Tuple*)closureType)->getTypes()[0]; + if (!interior->getTypes().size() || interior->getNames().back() != " _globals") { + throw std::runtime_error("Invalid closure-enclosing-globals"); + } + + Instance i = ( + ClosureVariableBinding() + 0 + ClosureVariableBindingStep(" _globals") + ).extractValueOrContainingClosure(closureType, closureData); + + if (i.type() != PythonObjectOfType::AnyPyDict()) { + throw std::runtime_error("Invalid closure-enclosing-globals"); + } + + funcGlobals.set(PythonObjectOfType::getPyObj(i.data())); + } + PyObject* res = PyFunction_New(mFunctionCode, (PyObject*)funcGlobals); if (!res) { diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index d38659664..3ed580df1 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -36,6 +36,7 @@ class FunctionOverload { PyObject* pyFuncAnnotations, const std::map& inGlobals, const std::vector& pyFuncClosureVarnames, + const std::vector& globalsInClosureVarnames, const std::map closureBindings, Type* returnType, PyObject* pySignatureFunction, @@ -47,6 +48,7 @@ class FunctionOverload { mFunctionAnnotations(pyFuncAnnotations), mGlobals(inGlobals), mFunctionClosureVarnames(pyFuncClosureVarnames), + mFunctionGlobalsInClosureVarnames(globalsInClosureVarnames), mReturnType(returnType), mSignatureFunction(pySignatureFunction), mArgs(args), @@ -94,6 +96,7 @@ class FunctionOverload { mMethodOf = other.mMethodOf; mFunctionClosureVarnames = other.mFunctionClosureVarnames; + mFunctionGlobalsInClosureVarnames = other.mFunctionGlobalsInClosureVarnames; mClosureBindings = other.mClosureBindings; mReturnType = other.mReturnType; @@ -124,6 +127,7 @@ class FunctionOverload { mFunctionAnnotations, mGlobals, mFunctionClosureVarnames, + mFunctionGlobalsInClosureVarnames, bindings, mReturnType, mSignatureFunction, @@ -139,6 +143,7 @@ class FunctionOverload { mFunctionAnnotations, mGlobals, mFunctionClosureVarnames, + mFunctionGlobalsInClosureVarnames, mClosureBindings, mReturnType, mSignatureFunction, @@ -154,6 +159,7 @@ class FunctionOverload { mFunctionAnnotations, mGlobals, mFunctionClosureVarnames, + mFunctionGlobalsInClosureVarnames, bindings, mReturnType, mSignatureFunction, @@ -231,6 +237,10 @@ class FunctionOverload { return mFunctionClosureVarnames; } + const std::vector& getFunctionGlobalsInClosureVarnames() const { + return mFunctionGlobalsInClosureVarnames; + } + const std::vector& getArgs() const { return mArgs; } @@ -624,10 +634,8 @@ class FunctionOverload { }); } - // get a list of all "global dot accesses" contained in 'code'. This will pull out every - // case where we have a sequence of opcodes that access a global variable by name and then - // sequentially access members. So if you write 'x.y.z' and 'x' is a reference that will - // be looked up using a 'LOAD_GLOBAL' then we will include 'x.y.z' in outAccesses. + // get a list of all "global accesses" contained in 'code'. This will pull out every + // case where we have an opcodes that access a global variable by name static void extractGlobalAccessesFromCode(PyCodeObject* code, std::set& outAccesses) { uint8_t* bytes; Py_ssize_t bytecount; @@ -792,6 +800,7 @@ class FunctionOverload { mMethodOf = other.mMethodOf; mFunctionClosureVarnames = other.mFunctionClosureVarnames; + mFunctionGlobalsInClosureVarnames = other.mFunctionGlobalsInClosureVarnames; mClosureBindings = other.mClosureBindings; mReturnType = other.mReturnType; @@ -829,6 +838,13 @@ class FunctionOverload { } buffer.writeEndCompound(); + buffer.writeBeginCompound(11); + stringIx = 0; + for (auto varname: mFunctionGlobalsInClosureVarnames) { + buffer.writeStringObject(stringIx++, varname); + } + buffer.writeEndCompound(); + buffer.writeBeginCompound(5); int varIx = 0; for (auto& nameAndGlobal: mGlobals) { @@ -876,6 +892,7 @@ class FunctionOverload { PyObjectHolder functionDefaults; PyObjectHolder functionSignature; std::vector closureVarnames; + std::vector functionGlobalsInClosureVarnames; std::map functionGlobals; std::map closureBindings; Type* returnType = nullptr; @@ -898,6 +915,12 @@ class FunctionOverload { closureVarnames.push_back(buffer.readStringObject()); }); } + else if (fieldNumber == 11) { + buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { + assertWireTypesEqual(wireType, WireType::BYTES); + functionGlobalsInClosureVarnames.push_back(buffer.readStringObject()); + }); + } else if (fieldNumber == 5) { std::string last; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { @@ -946,6 +969,7 @@ class FunctionOverload { functionAnnotations, functionGlobals, closureVarnames, + functionGlobalsInClosureVarnames, closureBindings, returnType, functionSignature, @@ -978,6 +1002,11 @@ class FunctionOverload { // function. This is the order that the python code will expect. std::vector mFunctionClosureVarnames; + // a list of variables which are globals, but which we are reading from the + // closure because the 'globals' dictionary is in fact a free python object + // that can't be pinned down in the type + std::vector mFunctionGlobalsInClosureVarnames; + PyObject* mFunctionDefaults; PyObject* mFunctionAnnotations; diff --git a/typed_python/ModuleRepresentationCopyContext.hpp b/typed_python/ModuleRepresentationCopyContext.hpp index a034e85dc..61b236480 100644 --- a/typed_python/ModuleRepresentationCopyContext.hpp +++ b/typed_python/ModuleRepresentationCopyContext.hpp @@ -230,6 +230,7 @@ class ModuleRepresentationCopyContext { copyObj(o.getFunctionAnnotations()), o.getGlobals(), o.getFunctionClosureVarnames(), + o.getFunctionGlobalsInClosureVarnames(), o.getClosureVariableBindings(), copyType(o.getReturnType()), o.getSignatureFunction(), diff --git a/typed_python/PyFunctionInstance.cpp b/typed_python/PyFunctionInstance.cpp index 1bc5cd921..6f51d6c73 100644 --- a/typed_python/PyFunctionInstance.cpp +++ b/typed_python/PyFunctionInstance.cpp @@ -1105,36 +1105,82 @@ void PyFunctionInstance::copyConstructFromPythonInstanceConcrete(Function* type, PyObject* pyClosure = PyFunction_GetClosure(pyRepresentation); - if (!pyClosure || !PyTuple_Check(pyClosure) || PyTuple_Size(pyClosure) != closureType->getTypes().size()) { - throw std::runtime_error("Expected the pyClosure to have " + format(closureType->getTypes().size()) + " cells."); + // our closure may or may not have a 'globals' dict at the end of it. + long expectedCellCount = closureType->getTypes().size(); + if (closureType->getTypes().size() && closureType->getTypes().back()->isPythonObjectOfType()) { + expectedCellCount -= 1; + } + + if (expectedCellCount == 0) { + if (pyClosure && !PyTuple_Check(pyClosure)) { + throw std::runtime_error("This PyFunction closure is not a tuple"); + } + + if (pyClosure && PyTuple_Size(pyClosure)) { + throw std::runtime_error( + "Our closure of type " + + closureType->name() + + " means we expect the python function's closure to have " + + format(expectedCellCount) + + " variables but it has " + format(PyTuple_Size(pyClosure)) + ); + } + } else { + if (!pyClosure || !PyTuple_Check(pyClosure) || PyTuple_Size(pyClosure) != expectedCellCount) { + throw std::runtime_error( + "Our closure of type " + + closureType->name() + + " means we expect the python function's closure to have " + + format(expectedCellCount) + + " variables but it has " + format(PyTuple_Size(pyClosure)) + ); + } } closureType->constructor(tgt, [&](instance_ptr tgtCell, int index) { Type* closureTypeInst = closureType->getTypes()[index]; - PyObject* cell = PyTuple_GetItem(pyClosure, index); - if (!cell) { - throw PythonExceptionSet(); - } + if (index < expectedCellCount) { + PyObject* cell = PyTuple_GetItem(pyClosure, index); + if (!cell) { + throw PythonExceptionSet(); + } - if (!PyCell_Check(cell)) { - throw std::runtime_error("Expected function closure to be made up of cells."); - } + if (!PyCell_Check(cell)) { + throw std::runtime_error("Expected function closure to be made up of cells."); + } - if (closureTypeInst->getTypeCategory() == Type::TypeCategory::catPyCell) { - // our representation in the closure is itself a PyCell, so we just reference - // the actual cell object. - static PyCellType* pct = PyCellType::Make(); - pct->initializeFromPyObject(tgtCell, cell); + if (closureTypeInst->getTypeCategory() == Type::TypeCategory::catPyCell) { + // our representation in the closure is itself a PyCell, so we just reference + // the actual cell object. + static PyCellType* pct = PyCellType::Make(); + pct->initializeFromPyObject(tgtCell, cell); + } else { + if (!PyCell_GET(cell)) { + throw std::runtime_error("Cell for " + closureType->getNames()[index] + " was empty."); + } + + PyInstance::copyConstructFromPythonInstance( + closureType->getTypes()[index], + tgtCell, + PyCell_GET(cell), + ConversionLevel::Implicit + ); + } } else { - if (!PyCell_GET(cell)) { - throw std::runtime_error("Cell for " + closureType->getNames()[index] + " was empty."); + PyObject* functionGlobals = PyFunction_GetGlobals(pyRepresentation); + if (!functionGlobals) { + throw std::runtime_error("Somehow this function didn't have globals?"); + } + + if (closureTypeInst != PythonObjectOfType::AnyPyDict()) { + throw std::runtime_error("Somehow the function closure type is improperly formed"); } PyInstance::copyConstructFromPythonInstance( closureType->getTypes()[index], tgtCell, - PyCell_GET(cell), + functionGlobals, ConversionLevel::Implicit ); } diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 6977db68e..b133ec2a2 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -1099,6 +1099,8 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { std::vector overloads; std::vector closureVarnames; + std::vector closureTupleVarnames; + std::vector globalsInClosureVarnames; std::vector closureVarTypes; std::map closureBindings; std::map globals; @@ -1129,6 +1131,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { throw PythonExceptionSet(); } closureVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); + closureTupleVarnames.push_back(std::string(PyUnicode_AsUTF8(varname))); } if (assumeClosureGlobal) { @@ -1152,11 +1155,41 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { } } - FunctionOverload::buildInitialGlobalsDict( - globals, - PyFunction_GetGlobals(funcObj), - (PyCodeObject*)PyFunction_GetCode(funcObj) - ); + if (!assumeClosureGlobal + && !CompilerVisibleObjectVisitor::isPyObjectGloballyIdentifiableModuleDict( + PyFunction_GetGlobals(funcObj) + ).size() + ) { + // this function has non-identifiable globals, so we're actually going to place the + // globals dict into the closure so that its part of the 'data' rather than the type + // of the class + closureVarTypes.push_back(PythonObjectOfType::AnyPyDict()); + // using a space in the name prevents this from colliding with an actual + // variable name + closureTupleVarnames.push_back(" _globals"); + + // ensure that every global lookup is now routed through the function overload + std::set allNamesString; + FunctionOverload::extractGlobalAccessesFromCode( + (PyCodeObject*)PyFunction_GetCode(funcObj), + allNamesString + ); + + for (auto name: allNamesString) { + globalsInClosureVarnames.push_back(name); + closureBindings[name] = + ClosureVariableBinding() + + 0 + + ClosureVariableBindingStep(" _globals") + + ClosureVariableBindingStep(name); + } + } else { + FunctionOverload::buildInitialGlobalsDict( + globals, + PyFunction_GetGlobals(funcObj), + (PyCodeObject*)PyFunction_GetCode(funcObj) + ); + } overloads.push_back( FunctionOverload( @@ -1165,6 +1198,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { ((PyFunctionObject*)(PyObject*)funcObj)->func_annotations, globals, closureVarnames, + globalsInClosureVarnames, closureBindings, rType, rSignature, @@ -1181,7 +1215,7 @@ PyObject *MakeFunctionType(PyObject* nullValue, PyObject* args) { Tuple::Make({ assumeClosureGlobal ? NamedTuple::Make({}, {}) : - NamedTuple::Make(closureVarTypes, closureVarnames) + NamedTuple::Make(closureVarTypes, closureTupleVarnames) }), false, false diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 7c3475bac..b06ea7695 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -10,6 +10,25 @@ from typed_python.test_util import CodeEvaluator +def test_function_in_anonymous_module_callable(): + c = CodeEvaluator() + m = {} + + c.evaluateInto(""" + from typed_python import Function + + y = 1 + + @Function + def f(x): + return x + y + """, m) + f = m['f'] + + assert f(10) == 11 + assert f.ClosureType.ElementTypes[0].ElementNames[0] == ' _globals' + + def test_identity_hash_of_alternative_stable(): e = CodeEvaluator() m = {} @@ -71,9 +90,9 @@ def f(self): assert C.MRO == (C,) assert CResolved.MRO == (CResolved,) - + assert CResolved.MemberNames == ('x',) - + assert C.MemberNames == ('x',) assert C.f.overloads[0].globals['C'].kind == 'GlobalInCell' diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 3be8dc45c..a5bea86dd 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2781,10 +2781,6 @@ def makeC(): " @Entrypoint\n" " def anF():\n" " return C\n" - "print(isForwardDefined(C))\n" - "print(forwardDefinitionsFor(C)[0].anF.overloads[0].globals)\n" - "print(isForwardDefined(forwardDefinitionsFor(C)[0]))\n" - "print(C.anF.overloads[0].globals)\n" ) with open(path, "w") as f: @@ -2800,8 +2796,6 @@ def makeC(): s = SerializationContext() return s.serialize(globals['C']) - makeC() - return serializedC = callFunctionInFreshProcess(makeC, ()) C = SerializationContext().deserialize(serializedC) From 5c2eb90416716af5e08d3c3aef821a73fb58c13e Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sat, 22 Jul 2023 14:35:51 +0000 Subject: [PATCH 55/83] Ensure we can compile things again. --- todo.txt | 7 +- typed_python/PyClassInstance.cpp | 2 +- typed_python/PyFunctionOverload.cpp | 13 ++ typed_python/TypedClosureBuilder.hpp | 2 +- typed_python/ValueType.cpp | 25 ++-- typed_python/ValueType.hpp | 2 +- typed_python/_types.cpp | 23 +-- .../compiler/expression_conversion_context.py | 14 +- .../compiler/python_to_native_converter.py | 137 +++--------------- .../python_typed_function_wrapper.py | 21 +-- .../compiler/type_wrappers/wrapper.py | 3 + typed_python/type_construction_test.py | 14 +- typed_python/types_serialization_test.py | 11 ++ 13 files changed, 102 insertions(+), 172 deletions(-) diff --git a/todo.txt b/todo.txt index 37c15ef12..5358d6853 100644 --- a/todo.txt +++ b/todo.txt @@ -25,7 +25,12 @@ - we need to make sure we can actually destroy these if we're not going to keep them - get rid of PyObject throughout Function - should be like a FunctionGlobal - +- Value should be holding a CompilerVisiblePyObj, not an instance +- We're using the TP compiler hash in the compiler, which makes sense. Need to ensure that + the CVOV obyes the same rules for dotted accesses that the original compiler does. +- We need to ensure that Globals retain their source information if its available + - this means that they need to allow themselves to be constants + - the whole 'globalsRaw' thing is trying to do this but its a crappy way of doing it def f(x): class C(Class): diff --git a/typed_python/PyClassInstance.cpp b/typed_python/PyClassInstance.cpp index a8e9a1287..e8744d8d2 100644 --- a/typed_python/PyClassInstance.cpp +++ b/typed_python/PyClassInstance.cpp @@ -676,7 +676,7 @@ PyObject* newPyTuple(const container_type& container, const func_type& objCreato PyObject* tup = PyTuple_New(container.size()); long ix = 0; for (auto containerElt: container) { - PyTuple_SetItem(tup, ix, convertPyTupleEltToPyObj(objCreator(containerElt))); + PyTuple_SetItem(tup, ix++, convertPyTupleEltToPyObj(objCreator(containerElt))); } return tup; } diff --git a/typed_python/PyFunctionOverload.cpp b/typed_python/PyFunctionOverload.cpp index 51f20754e..0ea4d32ae 100644 --- a/typed_python/PyFunctionOverload.cpp +++ b/typed_python/PyFunctionOverload.cpp @@ -135,6 +135,18 @@ void PyFunctionOverload::initFields() { PyDict_SetItemString(pyClosureVarsDict, nameAndClosureVar.first.c_str(), bindingObj); } + PyObjectStealer pyClosureVarnames( + PyTuple_New(overload.getFunctionClosureVarnames().size()) + ); + + for (long k = 0; k < overload.getFunctionClosureVarnames().size(); k++) { + PyTuple_SetItem( + pyClosureVarnames, + k, + PyUnicode_FromString(overload.getFunctionClosureVarnames()[k].c_str()) + ); + } + PyObjectStealer emptyTup(PyTuple_New(0)); PyObjectStealer emptyDict(PyDict_New()); @@ -162,6 +174,7 @@ void PyFunctionOverload::initFields() { PyDict_SetItemString(mDict, "index", (PyObject*)PyLong_FromLong(mOverloadIx)); PyDict_SetItemString(mDict, "closureVarLookups", (PyObject*)pyClosureVarsDict); + PyDict_SetItemString(mDict, "closureVars", (PyObject*)pyClosureVarnames); PyDict_SetItemString(mDict, "functionCode", (PyObject*)overload.getFunctionCode()); PyDict_SetItemString(mDict, "globals", (PyObject*)pyFunctionGlobals); PyDict_SetItemString(mDict, "returnType", overload.getReturnType() ? (PyObject*)PyInstance::typePtrToPyTypeRepresentation(overload.getReturnType()) : Py_None); diff --git a/typed_python/TypedClosureBuilder.hpp b/typed_python/TypedClosureBuilder.hpp index 6dd70846b..d4dfd0291 100644 --- a/typed_python/TypedClosureBuilder.hpp +++ b/typed_python/TypedClosureBuilder.hpp @@ -408,7 +408,7 @@ class TypedClosureBuilder { } PyObject* buildFinalResult() { - Function* outType = (Function*)mResolvedFunctionTypes[Path()]; + Function* outType = (Function*)mResolvedFunctionTypes[Path()]->forwardResolvesTo(); if (!outType) { throw std::runtime_error("Somehow, we don't have a function at the root of the closure converter"); diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 2d04abb60..57bb9d666 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -25,21 +25,14 @@ Value::Value() : } -Value::Value(const Instance& instance) : +Value::Value(const Instance& instance, PyObject* valueAsPyobj) : Type(TypeCategory::catValue), mInstance(instance), - mValueAsPyobj(nullptr) + mValueAsPyobj(incref(valueAsPyobj)) { m_size = 0; m_is_default_constructible = true; - mValueAsPyobj = PyInstance::extractPythonObject(mInstance); - - if (!mValueAsPyobj) { - PyErr_Clear(); - throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); - } - m_is_forward_defined = true; recomputeName(); } @@ -66,9 +59,9 @@ Type* Value::Make(Instance i) { } if (t->isForwardDefined()) { - typeMemo[t] = new Value(i); + typeMemo[t] = new Value(i, obj); } else { - typeMemo[t] = (Value*)(new Value(i))->forwardResolvesTo(); + typeMemo[t] = (Value*)(new Value(i, obj))->forwardResolvesTo(); } return typeMemo[t]; @@ -80,7 +73,13 @@ Type* Value::Make(Instance i) { auto it = instanceMemo.find(i); if (it == instanceMemo.end()) { - Value* resolved = (Value*)((new Value(i))->forwardResolvesTo()); + PyObject* obj = PyInstance::extractPythonObject(i); + + if (!obj) { + throw std::runtime_error("Failed to convert an instance of type " + i.type()->name() + " to a PyObject!"); + } + + Value* resolved = (Value*)((new Value(i, obj))->forwardResolvesTo()); it = instanceMemo.insert(std::make_pair(i, resolved)).first; } @@ -131,7 +130,9 @@ void Value::updateInternalTypePointersConcrete( mInstance = Instance::create( (PyObject*)PyInstance::typeObj(it->second) ); + mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + if (!mValueAsPyobj) { PyErr_Clear(); throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index a65c7070f..4818c16ae 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -24,7 +24,7 @@ PyDoc_STRVAR(Value_doc, class Value : public Type { public: - Value(const Instance& instance); + Value(const Instance& instance, PyObject* valueAsPyobj); // this is the clone-pathway Value(); diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index b133ec2a2..59aaf0295 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -2768,20 +2768,21 @@ PyObject *forwardDefinitionsFor(PyObject* nullValue, PyObject* args) { } PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { - if (PyTuple_Size(args) != 1) { - PyErr_SetString(PyExc_TypeError, "resolveForwardDefinedType takes 1 positional argument"); - return NULL; - } - PyObjectHolder a1(PyTuple_GetItem(args, 0)); + return ::translateExceptionToPyObject([&]() { + if (PyTuple_Size(args) != 1) { + throw std::runtime_error("resolveForwardDefinedType takes 1 positional argument"); + } + + PyObjectHolder a1(PyTuple_GetItem(args, 0)); - Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); + Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); - if (!t) { - PyErr_SetString(PyExc_TypeError, "first argument to 'resolveForwardDefinedType' must be a type object"); - return NULL; - } + if (!t) { + throw std::runtime_error( + "first argument to 'resolveForwardDefinedType' must be a type object" + ); + } - return ::translateExceptionToPyObject([&]() { return incref( (PyObject*) PyInstance::typeObj( diff --git a/typed_python/compiler/expression_conversion_context.py b/typed_python/compiler/expression_conversion_context.py index 07e098f22..8e3404909 100644 --- a/typed_python/compiler/expression_conversion_context.py +++ b/typed_python/compiler/expression_conversion_context.py @@ -1161,12 +1161,7 @@ def call_py_function(self, f, args, kwargs, returnTypeOverload=None): globalsInCells.append(f.__code__.co_freevars[i]) call_target = self.functionContext.converter.convert( - f.__name__, - f.__code__, - funcGlobals, - f.__globals__, - globalsInCells, - [], + typedFunc.overloads[0], [a.expr_type for a in concreteArgs], returnTypeOverload ) @@ -1205,12 +1200,7 @@ def call_overload(self, overload, funcObj, args, kwargs, returnTypeOverload=None returnType = typeWrapper(overload.returnType) if overload.returnType is not None else None call_target = self.functionContext.converter.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - list(overload.funcGlobalsInCells), - list(overload.closureVarLookups), + overload, [a.expr_type for a in closureArgs] + [a.expr_type for a in concreteArgs], returnType diff --git a/typed_python/compiler/python_to_native_converter.py b/typed_python/compiler/python_to_native_converter.py index e385f1503..1b31927ea 100644 --- a/typed_python/compiler/python_to_native_converter.py +++ b/typed_python/compiler/python_to_native_converter.py @@ -134,28 +134,24 @@ def identityHashToLinkerName(self, name, compilerHash, prefix="tp."): def createConversionContext( self, identity, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - closureVars, + overload, input_types, output_type, conversionType ): ConverterType = conversionType or FunctionConversionContext - pyast = self._code_to_ast(funcCode) + pyast = self._code_to_ast(overload.functionCode) return ConverterType( self, - funcName, + overload.name, identity, input_types, output_type, - closureVars, - funcGlobals, - funcGlobalsRaw, + overload.closureVarLookups, + overload.realizedGlobals, + overload.globals, pyast.args, pyast, ) @@ -591,103 +587,27 @@ def convertTypedFunctionCall(self, functionType, overloadIx, inputWrappers, asse returnType = None return self.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.globals, - {}, - list(overload.closureVarLookups), + overload, realizedInputWrappers, returnType, assertIsRoot=assertIsRoot ) - def hashObjectToIdentity(self, hashable, isModuleVal=False): - if isinstance(hashable, Hash): - return hashable - - if isinstance(hashable, int): - return Hash.from_integer(hashable) - - if isinstance(hashable, str): - return Hash.from_string(hashable) - - if hashable is None: - return Hash.from_integer(1) + Hash.from_integer(0) - - if isinstance(hashable, (dict, list)) and isModuleVal: - # don't look into dicts and lists at module level - return Hash.from_integer(2) - - if isinstance(hashable, (tuple, list)): - res = Hash.from_integer(len(hashable)) - for t in hashable: - res += self.hashObjectToIdentity(t, isModuleVal) - return res - - if isinstance(hashable, Wrapper): - return hashable.compilerHash() - - return Hash(_types.compilerHash(hashable)) - - def hashGlobals(self, funcGlobals, code, funcGlobalsFromCells): - """Hash a given piece of code's accesses to funcGlobals. - - We're trying to make sure that if we have a reference to module 'x' - in our globals, but we only ever use 'x' by writing 'x.f' or 'x.g', then - we shouldn't depend on the entirety of the definition of 'x'. - """ - - res = Hash.from_integer(0) - - for dotSeq in _types.getCodeGlobalDotAccesses(code): - res += self.hashDotSeq(dotSeq, funcGlobals) - - for globalName in funcGlobalsFromCells: - res += self.hashDotSeq([globalName], funcGlobals) - - return res - - def hashDotSeq(self, dotSeq, funcGlobals): - if not dotSeq or dotSeq[0] not in funcGlobals: - return Hash.from_integer(0) - - item = funcGlobals[dotSeq[0]] - - if not isinstance(item, ModuleType) or len(dotSeq) == 1: - return Hash.from_string(dotSeq[0]) + self.hashObjectToIdentity(item, True) - - if not hasattr(item, dotSeq[1]): - return Hash.from_integer(0) - - return Hash.from_string(dotSeq[0] + "." + dotSeq[1]) + self.hashObjectToIdentity(getattr(item, dotSeq[1]), True) - def convert( self, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - funcGlobalsFromCells, - closureVars, + overload, input_types, output_type, assertIsRoot=False, conversionType=None ): - """Convert a single pure python function using args of 'input_types'. + """Convert a single PyFunctionOverload using args of 'input_types'. It will return no more than 'output_type'. if output_type is None we produce the tightest output type possible. Args: - funcName - the name of the function - funcCode - a Code object representing the code to compile - funcGlobals - the globals object from the relevant function - funcGlobalsRaw - the original globals object (with no merging or filtering done) - which we use to figure out the location of global variables that are not in cells. - funcGlobalsFromCells - a list of the names that are globals that are actually accessed - as cells. + overload - a PyFunctionOverload instance we want to compile input_types - a type for each free variable in the function closure, and then again for each input argument output_type - the output type of the function, if known. if this is None, @@ -698,23 +618,14 @@ def convert( conversionType - if None, this is a normal function conversion. Otherwise, this must be a subclass of FunctionConversionContext """ - assert isinstance(funcName, str) - assert isinstance(funcCode, types.CodeType) - assert isinstance(funcGlobals, dict) + funcName = overload.name input_types = tuple([typedPythonTypeToTypeWrapper(i) for i in input_types]) compilerHash = ( Hash.from_integer(1) - + self.hashObjectToIdentity(( - funcCode, - funcName, - input_types, - output_type, - closureVars, - conversionType - )) + - self.hashGlobals(funcGlobals, funcCode, funcGlobalsFromCells) + + Hash.from_integer(overload.index) + + Hash(_types.compilerHash(overload.functionTypeObject)) ) assert not compilerHash.isPoison() @@ -729,7 +640,11 @@ def convert( if identity not in self._identifier_to_pyfunc: self._identifier_to_pyfunc[identity] = ( - funcName, funcCode, funcGlobals, closureVars, input_types, output_type, conversionType + overload.functionTypeObject, + overload.index, + input_types, + output_type, + conversionType ) isRoot = len(self._inflight_function_conversions) == 0 @@ -759,11 +674,7 @@ def convert( if identity not in self._inflight_function_conversions: functionConverter = self.createConversionContext( identity, - funcName, - funcCode, - funcGlobals, - funcGlobalsRaw, - closureVars, + overload, input_types, output_type, conversionType @@ -820,19 +731,17 @@ def _installInflightFunctions(self): if identifier in self._identifier_to_pyfunc: for v in self._visitors: - funcName, funcCode, funcGlobals, closureVars, input_types, output_type, conversionType = ( + functionTypeObject, overloadIx, input_types, output_type, conversionType = ( self._identifier_to_pyfunc[identifier] ) - + overload = functionTypeObject.overloads[overloadIx] + try: v.onNewFunction( identifier, functionConverter, nativeFunction, - funcName, - funcCode, - funcGlobals, - closureVars, + overload, input_types, functionConverter._varname_to_type.get(FunctionOutput), functionConverter._varname_to_type.get(FunctionYield), diff --git a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py index dc709349b..f2027eed8 100644 --- a/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py +++ b/typed_python/compiler/type_wrappers/python_typed_function_wrapper.py @@ -347,12 +347,7 @@ def convert_comprehension(self, context, instance, ConvertionContextType): # there's definitely a better way to organize this code. singleConvertedOverload = context.functionContext.converter.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - list(overload.funcGlobalsInCells), - list(overload.closureVarLookups), + overload, [typeWrapper(self.closurePathToCellType(path, closureType)) for path in overload.closureVarLookups.values()], None, conversionType=ConvertionContextType @@ -512,12 +507,7 @@ def convert_call(self, context, left, args, kwargs): # just one overload will do. We can just instantiate this particular function # with a signature that comes from the method overload signature itself. singleConvertedOverload = context.functionContext.converter.convert( - overload.name, - overload.functionCode, - overload.realizedGlobals, - overload.functionGlobals, - list(overload.funcGlobalsInCells), - list(overload.closureVarLookups), + overload, [typeWrapper(self.closurePathToCellType(path, closureType)) for path in overload.closureVarLookups.values()] + [a.expr_type for a in argsToPass], returnType @@ -734,12 +724,7 @@ def determinePossibleReturnTypes(converter, func, argTypes, kwargTypes): ) callTarget = converter.convert( - o.name, - o.functionCode, - o.realizedGlobals, - o.functionGlobals, - list(o.funcGlobalsInCells), - list(o.closureVarLookups), + o, [typeWrapper(PythonTypedFunctionWrapper.closurePathToCellType(path, func.ClosureType)) for path in o.closureVarLookups.values()] + actualArgTypes, None diff --git a/typed_python/compiler/type_wrappers/wrapper.py b/typed_python/compiler/type_wrappers/wrapper.py index fb1e5f84d..ede3c5e45 100644 --- a/typed_python/compiler/type_wrappers/wrapper.py +++ b/typed_python/compiler/type_wrappers/wrapper.py @@ -79,6 +79,9 @@ class Wrapper: # is this wrapping a OneOf object is_oneof_wrapper = False + def __reduce__(self): + return (typeWrapper, (self.typeRepresentation,)) + def __repr__(self): return "Wrapper(%s)" % str(self) diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index b06ea7695..ccaa4b80d 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -4,7 +4,7 @@ TupleOf, ListOf, OneOf, Forward, isForwardDefined, bytecount, resolveForwardDefinedType, Tuple, NamedTuple, PointerTo, RefTo, Dict, Set, Alternative, Function, identityHash, Value, Class, Member, ConstDict, TypedCell, typeLooksResolvable, Held, NotCompiled, - forwardDefinitionsFor + forwardDefinitionsFor, Entrypoint ) from typed_python.test_util import CodeEvaluator @@ -29,6 +29,18 @@ def f(x): assert f.ClosureType.ElementTypes[0].ElementNames[0] == ' _globals' +def test_entrypoint_basic(): + @Entrypoint + def f(x: int): + return x + 1 + + @Entrypoint + def g(): + return f(2) + + assert g() == 3 + + def test_identity_hash_of_alternative_stable(): e = CodeEvaluator() m = {} diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index a5bea86dd..77d1f1f08 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -1437,6 +1437,17 @@ def f(x: B = C) -> B: self.assertEqual(f(), f2()) + def test_serialize_function_types(self): + @Function + def f(x: int): + return x + + sc = SerializationContext() + + fType = sc.deserialize(sc.serialize(type(f))) + + assert fType is type(f) + def test_serialize_typed_classes(self): sc = SerializationContext() From b8b64a3c3be953bdb178760e963bbab3a5d6193d Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 24 Jul 2023 13:21:21 +0000 Subject: [PATCH 56/83] [wip] can serialize ForwardDefined types now --- todo.txt | 24 +++++-- typed_python/ForwardType.hpp | 32 ++++----- typed_python/PythonSerializationContext.hpp | 2 +- ...onSerializationContext_deserialization.cpp | 72 ++++++++++++++++--- ...thonSerializationContext_serialization.cpp | 49 ++++++++++--- typed_python/types_serialization_test.py | 38 +++++++++- 6 files changed, 178 insertions(+), 39 deletions(-) diff --git a/todo.txt b/todo.txt index 5358d6853..4b2f81fbf 100644 --- a/todo.txt +++ b/todo.txt @@ -1,12 +1,11 @@ -- ensure classes get a Python reference to their held functions - - all Forward defined types need to expose as much of their internals as possible - - not bytecounts or anything like that +- all Forward defined types need to expose as much of their internals as are known + - not bytecounts - this should be explicitly disallowed - expose code to allow us to get the _first_ forward that defined a given class - dotted globals? - if we do this, how do we resolve the dotted global value? - this is different for identity hashing than it is for serialization? - what were we doing in the existing system? This is very squirrely and difficult. -- fix the way we walk things in MRTG - should be unified +- fix the way we walk things in MRTG - should be unified so that we actually can see CVPOs - ensure that TypeFunction works as intended now. Some of the weird issues should be resolved now - fix the ModuleRepresentation stuff - redocument everything @@ -18,6 +17,20 @@ are not - can we get rid of the whole serialize-mrtg thing? at this point, memos should just work right? - this might leak which would be a problem + - should CVPOs be shared_ptr? we can still create cycles in them +- fill out a full CVPO model: + - classes + - tuples + - lists and other py datastructures + - hash their original contents. they are not supposed to change +- function objects + - default arguments need to be part of the instance closure if they have state? +- build a formal model for module objects and their contents + - the only reassignment we should allow is for a module member to go from unresolved to resolved + - nothing should change after import + - this would let us detect module changes + - we could serialize/deserialize without hitting the GIL + - as part of this, move more of the serialization logic to C++ which would be faster - ensure that we don't try to MRTG something that's a forward defined type - fix the compiler to deal with the new globals thing now that we have it well defined - figure out how to properly internalize python classes @@ -31,6 +44,9 @@ - We need to ensure that Globals retain their source information if its available - this means that they need to allow themselves to be constants - the whole 'globalsRaw' thing is trying to do this but its a crappy way of doing it +- take a look at object identity + - we could keep a reverse lookup from all alive python objects to the original instance + for things like lists/classes/etc def f(x): class C(Class): diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index bf1d38701..37801b7dd 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -42,22 +42,6 @@ class Forward : public Type { return Forward_doc; } - std::string moduleNameConcrete() { - if (mTarget) { - return mTarget->moduleNameConcrete(); - } - - return ""; - } - - std::string nameWithModuleConcrete() { - if (mTarget) { - return mTarget->nameWithModule(); - } - - return m_name; - } - static Forward* Make() { return Make("unnamed"); } @@ -146,6 +130,22 @@ class Forward : public Type { return mCellOrDict != nullptr; } + PyObject* getCellOrDict() { + return mCellOrDict; + } + + void setName(std::string newName) { + m_name = newName; + m_stripped_name = ""; + } + + void setCellOrDict(PyObject* newCellOrDict) { + if (mCellOrDict) { + decref(mCellOrDict); + } + mCellOrDict = incref(newCellOrDict); + } + bool lambdaDefinitionPopulated(); Type* lambdaDefinition(); diff --git a/typed_python/PythonSerializationContext.hpp b/typed_python/PythonSerializationContext.hpp index a0ababe4b..003505bfa 100644 --- a/typed_python/PythonSerializationContext.hpp +++ b/typed_python/PythonSerializationContext.hpp @@ -145,7 +145,7 @@ class PythonSerializationContext : public SerializationContext { void serializeNativeType(Type* nativeType, SerializationBuffer& b, size_t fieldNumber) const; - void serializeNativeTypeInner(Type* nativeType, SerializationBuffer& b, size_t fieldNumber) const; + void serializeNativeTypeInner(Type* nativeType, SerializationBuffer& b, size_t fieldNumber, bool checkIfObjectIsNamed) const; void serializeClassMembers(const std::vector& members, SerializationBuffer& b, int fieldNumber) const; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 3601f7514..009fbfb4c 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -629,7 +629,7 @@ Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::Typ result = new Function(); } if (typeCat == Type::TypeCategory::catForward) { - throw std::runtime_error("Can't deserialize actual forwards!"); + result = new Forward(""); } if (typeCat == Type::TypeCategory::catSet) { result = new SetType(); @@ -1275,10 +1275,12 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b MutuallyRecursiveTypeGroup* group = nullptr; std::string name; PyObjectHolder typeFromRep; + int64_t memo = -1; int32_t indexInGroup = -1; int32_t kind = -1; int32_t which = -1; int32_t category = -1; + Type* forwardType = nullptr; b.consumeCompoundMessage(inWireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { @@ -1310,6 +1312,39 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b if (kind == 4 && fieldNumber == 2) { assertWireTypesEqual(wireType, WireType::VARINT); indexInGroup = b.readUnsignedVarint(); + } else + if (kind == 5 && fieldNumber == 1) { + assertWireTypesEqual(wireType, WireType::VARINT); + memo = b.readUnsignedVarint(); + forwardType = (Type*)b.lookupCachedPointer(memo); + } else + if (kind == 5 && fieldNumber == 2) { + assertWireTypesEqual(wireType, WireType::VARINT); + category = b.readUnsignedVarint(); + } else + if (kind == 5 && fieldNumber == 3) { + assertWireTypesEqual(wireType, WireType::BYTES); + name = b.readStringObject(); + } else + if (kind == 5 && fieldNumber == 4) { + if (memo == -1) { + throw std::runtime_error("Invalid forward type serialization: expected a memo"); + } + if (category == -1) { + throw std::runtime_error("Invalid forward type serialization: expected a category"); + } + if (forwardType) { + throw std::runtime_error("Invalid forward type serialization: type is double-defined"); + } + forwardType = constructBlankSubclassOfTypeCategory(Type::TypeCategory(category)); + b.addCachedPointer(memo, (void*)forwardType, nullptr); + deserializeNativeTypeIntoBlank(b, wireType, forwardType, name); + + // at this point, we can finalize the type + forwardType->recomputeName(); + forwardType->finalizeType(); + forwardType->typeFinishedBeingDeserializedPhase1(); + forwardType->typeFinishedBeingDeserializedPhase2(); } else { throw std::runtime_error("Invalid nativeType: kind/fieldNumber error."); } @@ -1319,6 +1354,13 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b throw std::runtime_error("Invalid native type: no 'kind' field"); } + if (kind == 5) { + if (!forwardType) { + throw std::runtime_error("Invalid native type: forward body never given"); + } + return forwardType; + } + if (kind == 0) { if (category == -1) { throw std::runtime_error("Invalid inline type."); @@ -1553,17 +1595,23 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( if (category == -1) { throw std::runtime_error("Corrupt native type found."); } - if (category == Type::TypeCategory::catForward - || category == Type::TypeCategory::catBoundMethod - ) { + if (category == Type::TypeCategory::catBoundMethod) { if (fieldNumber == 1) { names.push_back(b.readStringObject()); } else if (fieldNumber == 2) { types.push_back(deserializeNativeType(b, wireType)); } } else - if (category == Type::TypeCategory::catAlternativeMatcher - ) { + if (category == Type::TypeCategory::catForward) { + if (fieldNumber == 1) { + names.push_back(b.readStringObject()); + } else if (fieldNumber == 2) { + types.push_back(deserializeNativeType(b, wireType)); + } else if (fieldNumber == 3) { + obj.steal(deserializePythonObject(b, wireType)); + } + } else + if (category == Type::TypeCategory::catAlternativeMatcher) { if (fieldNumber == 1) { types.push_back(deserializeNativeType(b, wireType)); } @@ -1720,13 +1768,21 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( ); } else if (category == Type::TypeCategory::catForward) { - if (names.size() != 1 || types.size() != 1) { + if (names.size() != 1 || types.size() > 1) { throw std::runtime_error("Corrupt Forward"); } if (!blankShell->isForward()) { throw std::runtime_error("Shell is not a Forward"); } - ((Forward*)blankShell)->define(types[0]); + + ((Forward*)blankShell)->setName(names[0]); + + if (obj) { + ((Forward*)blankShell)->setCellOrDict(obj); + } + if (types.size()) { + ((Forward*)blankShell)->define(types[0]); + } } else if (category == Type::TypeCategory::catHeldClass) { if (names.size() != 1) { diff --git a/typed_python/PythonSerializationContext_serialization.cpp b/typed_python/PythonSerializationContext_serialization.cpp index b059e7fd8..87e5db1e7 100644 --- a/typed_python/PythonSerializationContext_serialization.cpp +++ b/typed_python/PythonSerializationContext_serialization.cpp @@ -347,7 +347,32 @@ void PythonSerializationContext::serializeNativeType(Type* nativeType, Serializa PyEnsureGilAcquired getTheGil; if (nativeType->isForwardDefined() || nativeType->isForward()) { - throw std::runtime_error("Can't serialize Forward or forward-defined types yet"); + // forward types are neither stateful nor interned. So, we can serialize them + // with a memo and not worry about needing to intern them at any point. + b.writeBeginCompound(fieldNumber); + + // the 5 indicates that this is a forward native type. + b.writeUnsignedVarintObject(0, 5); + + uint32_t id; + bool isNew; + std::tie(id, isNew) = b.cachePointer(nativeType, nullptr); + + // write the memo first + b.writeUnsignedVarintObject(1, id); + + if (isNew) { + // then write the actual object if it's the first + // time we're writing it. note we write the type category first + // so that we can produce a blank instance before proceeding into + // the recursion + b.writeUnsignedVarintObject(2, nativeType->getTypeCategory()); + b.writeStringObject(3, nativeType->name()); + serializeNativeTypeInner(nativeType, b, 4, false); + } + + b.writeEndCompound(); + return; } # if TP_VERBOSE_SERIALIZE @@ -717,7 +742,7 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec } } else { Type* toSerialize = obj.type(); - serializeNativeTypeInner(toSerialize, b, index); + serializeNativeTypeInner(toSerialize, b, index, true); } }; @@ -820,7 +845,8 @@ void PythonSerializationContext::serializeMutuallyRecursiveTypeGroup(MutuallyRec void PythonSerializationContext::serializeNativeTypeInner( Type* nativeType, SerializationBuffer& b, - size_t fieldNumber + size_t fieldNumber, + bool checkIfObjectIsNamed ) const { b.writeBeginCompound(fieldNumber); @@ -836,14 +862,16 @@ void PythonSerializationContext::serializeNativeTypeInner( << nativeType->name() << " of cat " << nativeType->getTypeCategoryString() << "\n"; # endif - PyEnsureGilAcquired acquireTheGil; + if (checkIfObjectIsNamed) { + PyEnsureGilAcquired acquireTheGil; - std::string nameForObject = getNameForPyObj((PyObject*)PyInstance::typeObj(nativeType)); + std::string nameForObject = getNameForPyObj((PyObject*)PyInstance::typeObj(nativeType)); - if (nameForObject.size()) { - b.writeStringObject(0, nameForObject); - b.writeEndCompound(); - return; + if (nameForObject.size()) { + b.writeStringObject(0, nameForObject); + b.writeEndCompound(); + return; + } } b.writeUnsignedVarintObject(0, nativeType->getTypeCategory()); @@ -944,6 +972,9 @@ void PythonSerializationContext::serializeNativeTypeInner( if (((Forward*)nativeType)->getTarget()) { serializeNativeType(((Forward*)nativeType)->getTarget(), b, 2); } + if (((Forward*)nativeType)->getCellOrDict()) { + serializePythonObject(((Forward*)nativeType)->getCellOrDict(), b, 3); + } } else if (nativeType->isClass()) { serializeNativeType(((Class*)nativeType)->getHeldClass(), b, 1); } else if (nativeType->isHeldClass()) { diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index 77d1f1f08..e9c16c654 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -51,7 +51,7 @@ from typed_python._types import ( refcount, isRecursive, compilerHash, buildPyFunctionObject, setFunctionClosure, typesAreEquivalent, recursiveTypeGroupDeepRepr, - recursiveTypeGroupRepr + recursiveTypeGroupRepr, isForwardDefined ) module_level_testfun = dummy_test_module.testfunction @@ -1448,6 +1448,42 @@ def f(x: int): assert fType is type(f) + def test_serialize_empty_forward(self): + F1 = Forward("F1") + F2 = Forward("F2") + F1.define(ListOf(F2)) + + assert F1.get().ElementType is F2 + assert F2.get() is None + + F1_copy, F2_copy = SerializationContext().deserialize( + SerializationContext().serialize((F1, F2)) + ) + + assert F1_copy is not F1 + assert F2_copy is not F2 + + assert F1_copy.get().ElementType is F2_copy + assert F2_copy.get() is None + assert F1.__name__ == F1_copy.__name__ + assert F2.__name__ == F2_copy.__name__ + + F2.define(OneOf(None, F1)) + F2_copy.define(OneOf(None, F1_copy)) + + print(F2.resolve()) + print(F2_copy.resolve()) + F2r = F2_copy.resolve() + + print(F2r.Types[1].ElementType.__typed_python_category__) + + assert F2_copy.get().Types[1].get().ElementType is F2_copy + assert F2r.Types[1].ElementType is F2_copy + assert isForwardDefined(F2r.Types[1]) + + # assert F1.resolve() is F1_copy.resolve() + # assert F2.resolve() is F2_copy.resolve() + def test_serialize_typed_classes(self): sc = SerializationContext() From 9e1bbc61d46e0807c3d40b34f03c4c2cc6f0a339 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 24 Jul 2023 23:02:46 +0000 Subject: [PATCH 57/83] more wip --- repro.py | 29 +++++++++----- todo.txt | 3 ++ typed_python/CompilerVisibleObjectVisitor.hpp | 8 +++- typed_python/FunctionGlobal.hpp | 7 ++++ typed_python/PythonSerializationContext.hpp | 2 +- ...onSerializationContext_deserialization.cpp | 12 +++--- typed_python/Type.hpp | 3 +- .../compiler/native_compiler/native_ast.py | 5 ++- .../compiler/python_to_native_converter.py | 38 ++++++++++++++++--- typed_python/types_serialization_test.py | 18 +++++---- 10 files changed, 91 insertions(+), 34 deletions(-) diff --git a/repro.py b/repro.py index 6582553cc..8aa7fb6bd 100644 --- a/repro.py +++ b/repro.py @@ -1,18 +1,28 @@ import sys -from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final, Function, ListOf, NotCompiled +from typed_python import Class, SerializationContext, Held, isForwardDefined, Entrypoint, Forward, Final, Function, ListOf, NotCompiled, Member from typed_python._types import typeWalkRecord, recursiveTypeGroupRepr +import typed_python.compiler.native_compiler.native_ast as n +from typed_python import _types +assert not _types.checkForHashInstability() + +print(n.Type.zero) +print(n.Type.zero.overloads) + def writer(): - @NotCompiled - def fn(x) -> str: - return str(x) + Cls = Forward("Cls") + + @Cls.define + class Cls(Class): + m = Member(str) - print(fn.overloads[0].globals) + def f(self) -> Cls: + return Cls(m='HI') - bytesToWrite = SerializationContext().serialize(fn) + bytesToWrite = SerializationContext().serialize((Cls, ())) with open("a.dat", "wb") as f: f.write(bytesToWrite) @@ -20,11 +30,10 @@ def fn(x) -> str: def reader(): with open("a.dat", "rb") as f: - x = SerializationContext().deserialize(f.read()) + Cls, args = SerializationContext().deserialize(f.read()) - print(x.__name__) - print(x.f.overloads[0].globals) - print(x().f().f()) + # print(x.__name__) + print(Cls(*args).f().m) if sys.argv[1:] == ['r']: diff --git a/todo.txt b/todo.txt index 4b2f81fbf..5e1a1a4a6 100644 --- a/todo.txt +++ b/todo.txt @@ -1,3 +1,6 @@ +- fully internalized types shouldn't be able to have an unstable identity at all + - NamedModuleMember should identity hash the same way regardless anyways + - we should force these Globals to resolve to something specific at resolution time - all Forward defined types need to expose as much of their internals as are known - not bytecounts - this should be explicitly disallowed - expose code to allow us to get the _first_ forward that defined a given class diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index d40e9fb0d..86868f0a8 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -739,7 +739,7 @@ class CompilerVisibleObjectVisitor { visitor.visitHash(ShaHash(2)); visitor.visitTopo(argType); visitor.visitInstance( - argType, + argType, ((PyInstance*)obj.pyobj())->dataPtr() ); return; @@ -894,6 +894,12 @@ class CompilerVisibleObjectVisitor { PyTypeObject* tp = (PyTypeObject*)obj.pyobj(); + if (tp->tp_name == std::string("typed_python.types_serialization_test.Cls")) { + asm("int3"); + } + + std::cout << tp->tp_name << "\n"; + visitor.visitHash(ShaHash(tp->tp_name)); visitor.visitHash(ShaHash(0)); diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 2d21cdbc4..2dbcd1f69 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -272,6 +272,13 @@ class FunctionGlobal { if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { PyObject* obj = extractGlobalRefFromDictOrCell(); + if (isNamedModuleMember() && mModuleName == "typed_python.compiler.native_compiler.native_ast" && mName == "Expression") { + std::cout << "Visiting it! " << (obj ? "not empty":"empty") << "\n"; + if (!obj) { + asm("int3"); + } + } + if (obj) { _visitCompilerVisibleInternalsInPyobj(obj, visitor); } diff --git a/typed_python/PythonSerializationContext.hpp b/typed_python/PythonSerializationContext.hpp index 003505bfa..14b96cf8b 100644 --- a/typed_python/PythonSerializationContext.hpp +++ b/typed_python/PythonSerializationContext.hpp @@ -163,7 +163,7 @@ class PythonSerializationContext : public SerializationContext { void deserializeClassClassMemberDict(std::map& dict, DeserializationBuffer& b, int wireType) const; - static Type* constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat); + static Type* constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat, bool isForwardDefined = false); void deserializeNativeTypeIntoBlank(DeserializationBuffer& b, size_t wireType, Type* blankShell, std::string inName) const; diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 009fbfb4c..f6be5ef68 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -574,7 +574,7 @@ PyObject* prepareBlankInstanceOfType(MutuallyRecursiveTypeGroup* outGroup, PyTyp } /* static */ -Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat) { +Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::TypeCategory typeCat, bool isForwardDefined) { Type* result = nullptr; if (typeCat == Type::TypeCategory::catValue) { @@ -645,7 +645,7 @@ Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::Typ } if (result) { - result->markActivelyBeingDeserialized(); + result->markActivelyBeingDeserialized(isForwardDefined); return result; } @@ -1316,7 +1316,7 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b if (kind == 5 && fieldNumber == 1) { assertWireTypesEqual(wireType, WireType::VARINT); memo = b.readUnsignedVarint(); - forwardType = (Type*)b.lookupCachedPointer(memo); + forwardType = (Type*)b.lookupCachedPointer(memo); } else if (kind == 5 && fieldNumber == 2) { assertWireTypesEqual(wireType, WireType::VARINT); @@ -1336,7 +1336,7 @@ Type* PythonSerializationContext::deserializeNativeType(DeserializationBuffer& b if (forwardType) { throw std::runtime_error("Invalid forward type serialization: type is double-defined"); } - forwardType = constructBlankSubclassOfTypeCategory(Type::TypeCategory(category)); + forwardType = constructBlankSubclassOfTypeCategory(Type::TypeCategory(category), true); b.addCachedPointer(memo, (void*)forwardType, nullptr); deserializeNativeTypeIntoBlank(b, wireType, forwardType, name); @@ -1774,9 +1774,9 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( if (!blankShell->isForward()) { throw std::runtime_error("Shell is not a Forward"); } - + ((Forward*)blankShell)->setName(names[0]); - + if (obj) { ((Forward*)blankShell)->setCellOrDict(obj); } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index f3fa85eb1..ac9be33d3 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -916,8 +916,9 @@ class Type { void attemptAutoresolveWrite(); - void markActivelyBeingDeserialized() { + void markActivelyBeingDeserialized(bool isForwardDefined) { m_is_being_deserialized = true; + m_is_forward_defined = isForwardDefined; } bool isActivelyBeingDeserialized() { diff --git a/typed_python/compiler/native_compiler/native_ast.py b/typed_python/compiler/native_compiler/native_ast.py index ea1a54d65..cfd48ab45 100644 --- a/typed_python/compiler/native_compiler/native_ast.py +++ b/typed_python/compiler/native_compiler/native_ast.py @@ -110,7 +110,6 @@ def typeZeroConstant(self): else False, zeroConstant=typeZeroConstant )) -Type = resolveForwardDefinedType(Type) def const_truth_value(c): @@ -159,7 +158,6 @@ def const_str(c): truth_value=const_truth_value, __str__=const_str )) -Constant = resolveForwardDefinedType(Constant) UnaryOp = Alternative( "UnaryOp", @@ -663,6 +661,9 @@ def expr_could_throw(self): )) +NamedCallTarget = resolveForwardDefinedType(NamedCallTarget) +Type = resolveForwardDefinedType(Type) +Constant = resolveForwardDefinedType(Constant) ExpressionIntermediate = resolveForwardDefinedType(ExpressionIntermediate) Expression = resolveForwardDefinedType(Expression) Teardown = resolveForwardDefinedType(Teardown) diff --git a/typed_python/compiler/python_to_native_converter.py b/typed_python/compiler/python_to_native_converter.py index 1b31927ea..e9597a841 100644 --- a/typed_python/compiler/python_to_native_converter.py +++ b/typed_python/compiler/python_to_native_converter.py @@ -89,6 +89,34 @@ def addVisitor(self, visitor): def removeVisitor(self, visitor): self._visitors.remove(visitor) + def hashObjectToIdentity(self, hashable, isModuleVal=False): + if isinstance(hashable, Hash): + return hashable + + if isinstance(hashable, int): + return Hash.from_integer(hashable) + + if isinstance(hashable, str): + return Hash.from_string(hashable) + + if hashable is None: + return Hash.from_integer(1) + Hash.from_integer(0) + + if isinstance(hashable, (dict, list)) and isModuleVal: + # don't look into dicts and lists at module level + return Hash.from_integer(2) + + if isinstance(hashable, (tuple, list)): + res = Hash.from_integer(len(hashable)) + for t in hashable: + res += self.hashObjectToIdentity(t, isModuleVal) + return res + + if isinstance(hashable, Wrapper): + return hashable.compilerHash() + + return Hash(_types.compilerHash(hashable)) + def identityToName(self, identity): """Convert a function identity to the link-time name for the function. @@ -640,10 +668,10 @@ def convert( if identity not in self._identifier_to_pyfunc: self._identifier_to_pyfunc[identity] = ( - overload.functionTypeObject, - overload.index, - input_types, - output_type, + overload.functionTypeObject, + overload.index, + input_types, + output_type, conversionType ) @@ -735,7 +763,7 @@ def _installInflightFunctions(self): self._identifier_to_pyfunc[identifier] ) overload = functionTypeObject.overloads[overloadIx] - + try: v.onNewFunction( identifier, diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index e9c16c654..a71e095cc 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -1468,21 +1468,20 @@ def test_serialize_empty_forward(self): assert F1.__name__ == F1_copy.__name__ assert F2.__name__ == F2_copy.__name__ + assert isForwardDefined(F1_copy) + assert isForwardDefined(F1_copy.get()) + assert isForwardDefined(F1_copy.get().ElementType) + F2.define(OneOf(None, F1)) F2_copy.define(OneOf(None, F1_copy)) - print(F2.resolve()) - print(F2_copy.resolve()) F2r = F2_copy.resolve() - print(F2r.Types[1].ElementType.__typed_python_category__) - assert F2_copy.get().Types[1].get().ElementType is F2_copy - assert F2r.Types[1].ElementType is F2_copy - assert isForwardDefined(F2r.Types[1]) + assert F2r.Types[1].ElementType is F2r - # assert F1.resolve() is F1_copy.resolve() - # assert F2.resolve() is F2_copy.resolve() + assert F1.resolve() is F1_copy.resolve() + assert F2.resolve() is F2_copy.resolve() def test_serialize_typed_classes(self): sc = SerializationContext() @@ -2308,6 +2307,9 @@ class Cls: def f(self) -> Cls: return "HI" + with open("a.dat", "wb") as f: + f.write(SerializationContext().serialize((Cls, ()))) + assert callFunctionInFreshProcess(Cls, ()).f() == "HI" def test_can_deserialize_untyped_forward_class_methods_2(self): From e11996944176daa11662822ba1be8482ae72706e Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 25 Jul 2023 15:25:39 +0000 Subject: [PATCH 58/83] TypeFunctions don't have weird closure problems anymore. __typed_python_template__ is a staticmethod. --- repro.py | 2 +- todo.txt | 11 +++++++++++ typed_python/CompilerVisibleObjectVisitor.hpp | 6 ------ typed_python/HeldClassType.cpp | 3 +-- typed_python/PyInstance.cpp | 16 ++++++++++++---- typed_python/__init__.py | 2 +- .../compiler/type_wrappers/class_wrapper.py | 2 +- typed_python/internals.py | 4 +++- typed_python/types_serialization_test.py | 3 ++- 9 files changed, 32 insertions(+), 17 deletions(-) diff --git a/repro.py b/repro.py index 8aa7fb6bd..86ede4efd 100644 --- a/repro.py +++ b/repro.py @@ -33,7 +33,7 @@ def reader(): Cls, args = SerializationContext().deserialize(f.read()) # print(x.__name__) - print(Cls(*args).f().m) + print(Cls(*args).f()) if sys.argv[1:] == ['r']: diff --git a/todo.txt b/todo.txt index 5e1a1a4a6..12bc87500 100644 --- a/todo.txt +++ b/todo.txt @@ -1,6 +1,8 @@ - fully internalized types shouldn't be able to have an unstable identity at all - NamedModuleMember should identity hash the same way regardless anyways - we should force these Globals to resolve to something specific at resolution time +- Class/HeldClass classMembers could still have forwards. These are PyObjects and need to + - be converted to Constants - all Forward defined types need to expose as much of their internals as are known - not bytecounts - this should be explicitly disallowed - expose code to allow us to get the _first_ forward that defined a given class @@ -51,6 +53,15 @@ - we could keep a reverse lookup from all alive python objects to the original instance for things like lists/classes/etc + +question: how should we handle regular python classes defined purely in forwards? +* they will trigger MutuallyRecursiveTypeGroup and identityHash, which should be fine +* we've said that Forwards shouldn't participate in MRTG +* I think we need to actually build the formal graph in CVPO, and this should proceed with standard + interning rules. This will allow us to completely remove the 'Forwards' from the object. + + + def f(x): class C(Class): def g(self): diff --git a/typed_python/CompilerVisibleObjectVisitor.hpp b/typed_python/CompilerVisibleObjectVisitor.hpp index 86868f0a8..221899015 100644 --- a/typed_python/CompilerVisibleObjectVisitor.hpp +++ b/typed_python/CompilerVisibleObjectVisitor.hpp @@ -894,12 +894,6 @@ class CompilerVisibleObjectVisitor { PyTypeObject* tp = (PyTypeObject*)obj.pyobj(); - if (tp->tp_name == std::string("typed_python.types_serialization_test.Cls")) { - asm("int3"); - } - - std::cout << tp->tp_name << "\n"; - visitor.visitHash(ShaHash(tp->tp_name)); visitor.visitHash(ShaHash(0)); diff --git a/typed_python/HeldClassType.cpp b/typed_python/HeldClassType.cpp index 7fae7a77f..85f2a917e 100644 --- a/typed_python/HeldClassType.cpp +++ b/typed_python/HeldClassType.cpp @@ -425,8 +425,7 @@ HeldClass* HeldClass::Make( for (auto nameAndCM: classMembers) { auto it = base->getOwnClassMembers().find(nameAndCM.first); - if (it != base->getOwnClassMembers().end() - && nameAndCM.first != "__typed_python_template__") { + if (it != base->getOwnClassMembers().end()) { // check that they're the same object if (it->second != nameAndCM.second && nameAndCM.first != "__qualname__" && nameAndCM.first != "__module__" diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index e604c1f5c..61c8a3abf 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -381,13 +381,21 @@ PyObject* PyInstance::tp_new(PyTypeObject *subtype, PyObject *args, PyObject *kw eltType->assertForwardsResolvedSufficientlyToInstantiate(); - if (eltType->isClass() && ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__") - != ((Class*)eltType)->getOwnClassMembers().end()) { - return PyObject_Call( - ((Class*)eltType)->getOwnClassMembers().find("__typed_python_template__")->second, + if (eltType->isClass() && ((Class*)eltType)->getOwnStaticFunctions().find("__typed_python_template__") + != ((Class*)eltType)->getOwnStaticFunctions().end()) { + auto matchedAndResult = PyFunctionInstance::tryToCallAnyOverload( + ((Class*)eltType)->getOwnStaticFunctions().find("__typed_python_template__")->second, + nullptr, + nullptr, args, kwds ); + + if (!matchedAndResult.first) { + throw std::runtime_error("Somehow, __typed_python_template__ didn't dispatch"); + } + + return matchedAndResult.second; } Instance inst(eltType, [&](instance_ptr tgt) { diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 822911f0f..62fb6bb09 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -38,7 +38,6 @@ ) from typed_python._types import bytecount, refcount from typed_python.module import Module -from typed_python.type_function import TypeFunction from typed_python.hash import sha_hash from typed_python.compiler.typeof import TypeOf from typed_python._types import ( @@ -76,6 +75,7 @@ _types._enableTypeAutoresolution(True) +from typed_python.type_function import TypeFunction from typed_python.SerializationContext import SerializationContext # noqa from typed_python.compiler.runtime import Entrypoint, Compiled, NotCompiled, Runtime # noqa diff --git a/typed_python/compiler/type_wrappers/class_wrapper.py b/typed_python/compiler/type_wrappers/class_wrapper.py index 083ab0aeb..2c9f3c4b5 100644 --- a/typed_python/compiler/type_wrappers/class_wrapper.py +++ b/typed_python/compiler/type_wrappers/class_wrapper.py @@ -141,7 +141,7 @@ def _classCouldBeInstanceOf(cls, other): return False for name in set(cls.ClassMembers) & set(other.ClassMembers): - if name not in ['__qualname__', '__name__', '__module__', '__typed_python_template__', '__classcell__']: + if name not in ['__qualname__', '__name__', '__module__', '__classcell__']: if cls.ClassMembers[name] is not other.ClassMembers[name]: return False diff --git a/typed_python/internals.py b/typed_python/internals.py index df2352695..f4694bf52 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -325,7 +325,9 @@ def __new__(cls, name, bases, namespace, **kwds): and issubclass(elt, typed_python._types.Function) ): if eltName == '__typed_python_template__': - classMembers[eltName] = elt + staticFunctions[eltName] = makeFunctionType( + eltName, elt, assumeClosuresGlobal=True + ) elif eltName not in memberFunctions: memberFunctions[eltName] = makeFunctionType( eltName, elt, classname=name, assumeClosuresGlobal=True diff --git a/typed_python/types_serialization_test.py b/typed_python/types_serialization_test.py index a71e095cc..f3eaa64fa 100644 --- a/typed_python/types_serialization_test.py +++ b/typed_python/types_serialization_test.py @@ -2303,7 +2303,8 @@ def test_can_deserialize_untyped_forward_class_methods_1(self): @Cls.define class Cls: - # recall that regular classes ignore their annotations + # recall that regular classes ignore their annotations. This one will be a + # 'Forward' holding Cls. def f(self) -> Cls: return "HI" From e295b72b0b1ce411d54a4e4d4827c10d1153dfc5 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 25 Jul 2023 22:06:10 +0000 Subject: [PATCH 59/83] more todo --- todo.txt | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/todo.txt b/todo.txt index 12bc87500..5b2dff43e 100644 --- a/todo.txt +++ b/todo.txt @@ -54,11 +54,33 @@ for things like lists/classes/etc -question: how should we handle regular python classes defined purely in forwards? -* they will trigger MutuallyRecursiveTypeGroup and identityHash, which should be fine -* we've said that Forwards shouldn't participate in MRTG +question: how should we handle regular python classes defined in forwards? +* they will trigger MutuallyRecursiveTypeGroup and identityHash, which should be fine. +* we've said that Forwards shouldn't participate in MRTG. + * This means we need to get a representation of the object that is completely external to 'forwards' + * we won't allow monkey patching. * I think we need to actually build the formal graph in CVPO, and this should proceed with standard - interning rules. This will allow us to completely remove the 'Forwards' from the object. +interning rules. This will allow us to completely remove the 'Forwards' from the object. + * when a forward gets defined and pointed at a python object, we'll allow a walk inside + * the question 'typeLooksResolvable' will be valid to ask of a Class that has forwards in it + * if we ask 'isForwardDefined' we'll have to walk in and see if it references any forwards + * this walk can't be cached until we make the model of the python object because the object is mutable + * this could be slow but at least it's simple +* if you think about it what we really have is a graph of regular mutable python objects, of which + forward defined types are just a special case. Periodically we walk this graph and check if + we can do a forward resolution at which point we make a formal model +* to do this correctly we need two things: + * a full model of python objects including Function and Class, PyList, PyTuple, etc. + * a way of associating the model to a python instance + * this association should be the one we use for MRTG of python instances that are interned - their shadows + should be what we see and talk about, not the instances themselves. + * the compiler should be using this, not the original objects + * every CVPO should have a 'canonical' instance + * every PyObj should have a mapping to the CVPO that represents it if its been forward resolved + * you can ask if a python object isForwardDefined + * function instances can be forward defined as well + * if you choose to resolve (or we autoresolve, or a TypeFunction forces resolution) then we build the CVPO mapping + * then CVPO can completely replace TOPO From 93e1c92a18858efedc9c9572b206558ab09646d8 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 26 Jul 2023 20:19:52 +0000 Subject: [PATCH 60/83] Build a full model of CVPO. CVPOs can model pretty much everything we would expect to see walking an object graph now. --- todo.txt | 8 + typed_python/CompilerVisiblePyObj.hpp | 997 ++++++++++++++++--- typed_python/FunctionGlobal.hpp | 29 +- typed_python/PyCompilerVisiblePyObj.cpp | 134 ++- typed_python/PyCompilerVisiblePyObj.hpp | 7 + typed_python/compiler_visible_py_obj_test.py | 188 ++++ 6 files changed, 1210 insertions(+), 153 deletions(-) create mode 100644 typed_python/compiler_visible_py_obj_test.py diff --git a/todo.txt b/todo.txt index 5b2dff43e..8c6316885 100644 --- a/todo.txt +++ b/todo.txt @@ -1,3 +1,11 @@ +- cvpo + - be able to re-instantiate objects if they don't exist + - be able to hash an MRTG a type/CVPO graph from both the compiler and identity perspective + - the 'type' interning system should handle both CVPO and Type now + - the act of walking types and CVPO should create the CVPOs as we go + - need an object to manage this - we should be able to delete these when done? + - when we duplicate the type object we'll produce a copy of the CVPO + - we need to be able to produce the actual objects - fully internalized types shouldn't be able to have an unstable identity at all - NamedModuleMember should identity hash the same way regardless anyways - we should force these Globals to resolve to something specific at resolution time diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 2fde27587..43e01679b 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -26,27 +26,70 @@ CompilerVisiblePyObject a representation of a python object that's owned by TypedPython. We hold these by pointer and leak them indiscriminately - like Type objects they're considered to be permanent -and singletonish +and singletonish. +They are intended to be a 'snapshot' of the state of a collection of python objects and +contain enough information for the compiler to use them to build compiled code. + +To the extent that they visit 'mutable' objects like a list (which might be contained within +a default argument), they will encode the state of the object when it was first seen. This +lets us determine if the object was modified (which would break compiler invariants) and also +gives us a self-consistent view of the world to compile against so we don't have state +changing underneath us. **********************************/ class CompilerVisiblePyObj { enum class Kind { // this should never be visible in a running program Uninitialized = 0, - // we're pointing back into a fully resolved Type - Type = 1, - // we're pointing into a TP instannce that doesn't reach - // a more complex object. It can have Type leaves in it - Instance = 2, - // this is a python tuple object - PyTuple = 3, + // a string held in mStringObject + String, + // we're pointing to a "canonical" python object that's visible from a module + // and that we don't want to look inside of. mName will contain the + // name, and mModuleName will contain the module. We will assume that this object + // is the same across program invocations (its a C function and we can't + // look inside of it) + NamedPyObject, + // we're pointing back into a typed_python Type held in mType. + Type, + // we're pointing into a TP instance that doesn't reach a more complex object. + // It can have Type leaves in it. It will be held in 'mInstance' + Instance, + // a python list, with elements in mElements + PyList, + // a python Dict, with values in mElements and keys in mKeys + PyDict, + // a python Dict, with values in mElements and keys in mKeys + PySet, + // a python tuple with elements in mElements + PyTuple, + // a vanilla python 'class' which must be constructible by calling 'type' + PyClass, + // a vanilla python object whose type is a PyClass + PyObject, + // a python function. mNamedElements will contain code, annotations, ec. + // mStringObject will contain the name. + PyFunction, + // a code object. mName + PyCodeObject, + PyCell, + // a module object that can be looked up by name. We don't walk into this + // from here to avoid making a snapshot of module objects while they're being imported + PyModule, + // a module object's dict. + PyModuleDict, + // the dict of a vanilla PyClass + PyClassDict, + PyStaticMethod, + PyClassMethod, + PyBoundMethod, + PyProperty, // the bailout pathway for cases we don't handle well. We should // assume that the compiler will treat this object as a plain // PyObject without looking inside of it, and the details of this // object are insufficient to differentiate two different Function // types that both refer to different ArbitraryPyObject instances. - ArbitraryPyObject = 4 + ArbitraryPyObject }; CompilerVisiblePyObj() : @@ -57,6 +100,13 @@ class CompilerVisiblePyObj { } public: + static CompilerVisiblePyObj* ForType(Type* t) { + CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + res->mKind = Kind::Type; + res->mType = t; + return res; + } + bool isUninitialized() const { return mKind == Kind::Uninitialized; } @@ -65,88 +115,275 @@ class CompilerVisiblePyObj { return mKind == Kind::Type; } + bool isNamedPyObject() const { + return mKind == Kind::NamedPyObject; + } + + bool isString() const { + return mKind == Kind::String; + } + bool isInstance() const { return mKind == Kind::Instance; } + bool isPyList() const { + return mKind == Kind::PyList; + } + + bool isPyDict() const { + return mKind == Kind::PyDict; + } + + bool isPySet() const { + return mKind == Kind::PySet; + } + bool isPyTuple() const { return mKind == Kind::PyTuple; } - bool isArbitraryPyObject() const { - return mKind == Kind::ArbitraryPyObject; + bool isPyClass() const { + return mKind == Kind::PyClass; } - static CompilerVisiblePyObj* Type(Type* t) { - CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + bool isPyFunction() const { + return mKind == Kind::PyFunction; + } - res->mKind = Kind::Type; - res->mType = t; + bool isPyCell() const { + return mKind == Kind::PyCell; + } - return res; + bool isPyObject() const { + return mKind == Kind::PyObject; } - static CompilerVisiblePyObj* Instance(Instance i) { - CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + bool isPyCodeObject() const { + return mKind == Kind::PyCodeObject; + } - res->mKind = Kind::Instance; - res->mInstance = i; + bool isPyModule() const { + return mKind == Kind::PyModule; + } - return res; + bool isArbitraryPyObject() const { + return mKind == Kind::ArbitraryPyObject; } - static CompilerVisiblePyObj* PyTuple() { - CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + std::string kindAsString() const { + if (mKind == Kind::Uninitialized) { + return "Uninitialized"; + } - res->mKind = Kind::PyTuple; + if (mKind == Kind::String) { + return "String"; + } - return res; - } + if (mKind == Kind::NamedPyObject) { + return "NamedPyObject"; + } - static CompilerVisiblePyObj* ArbitraryPyObject(PyObject* val) { - CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + if (mKind == Kind::Type) { + return "Type"; + } + + if (mKind == Kind::Instance) { + return "Instance"; + } + + if (mKind == Kind::PyList) { + return "PyList"; + } - res->mKind = Kind::ArbitraryPyObject; - res->mPyObject = incref(val); + if (mKind == Kind::PyDict) { + return "PyDict"; + } - return res; + if (mKind == Kind::PySet) { + return "PySet"; + } + + if (mKind == Kind::PyTuple) { + return "PyTuple"; + } + + if (mKind == Kind::PyClass) { + return "PyClass"; + } + + if (mKind == Kind::PyFunction) { + return "PyFunction"; + } + + if (mKind == Kind::PyObject) { + return "PyObject"; + } + + if (mKind == Kind::PyCell) { + return "PyCell"; + } + + if (mKind == Kind::PyCodeObject) { + return "PyCodeObject"; + } + + if (mKind == Kind::PyModule) { + return "PyModule"; + } + + if (mKind == Kind::PyModuleDict) { + return "PyModuleDict"; + } + + if (mKind == Kind::PyClassDict) { + return "PyClassDict"; + } + + if (mKind == Kind::PyStaticMethod) { + return "PyStaticMethod"; + } + + if (mKind == Kind::PyClassMethod) { + return "PyClassMethod"; + } + + if (mKind == Kind::PyBoundMethod) { + return "PyBoundMethod"; + } + + if (mKind == Kind::PyProperty) { + return "PyProperty"; + } + + if (mKind == Kind::ArbitraryPyObject) { + return "ArbitraryPyObject"; + } + + throw std::runtime_error("Unknown CompilerVisiblePyObj Kind"); } - static CompilerVisiblePyObj* PyTuple(const std::vector& elts) { - CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + static std::string dictGetStringOrEmpty(PyObject* dict, const char* name) { + if (!dict || !PyDict_Check(dict)) { + return ""; + } + + PyObject* o = PyDict_GetItemString(dict, name); + if (!o || !PyUnicode_Check(o)) { + return ""; + } - res->mKind = Kind::PyTuple; - res->mElements = elts; + return PyUnicode_AsUTF8(o); + } - return res; + static std::string stringOrEmpty(PyObject* o) { + if (!o || !PyUnicode_Check(o)) { + return ""; + } + + return PyUnicode_AsUTF8(o); } - // return a CVPO for 'val', stashing it in 'constantMapCache' + // return a CVPO for 'val', stashing it in 'constantMapCache' // in case we hit a recursion. static CompilerVisiblePyObj* internalizePyObj( PyObject* val, std::unordered_map& constantMapCache, - const std::map<::Type*, ::Type*>& groupMap + const std::map<::Type*, ::Type*>& groupMap, + bool linkBackToOriginalObject = true ) { auto it = constantMapCache.find(val); - + if (it != constantMapCache.end()) { return it->second; } constantMapCache[val] = new CompilerVisiblePyObj(); - constantMapCache[val]->becomeInternalizedOf(val, constantMapCache, groupMap); + constantMapCache[val]->becomeInternalizedOf( + val, constantMapCache, groupMap, linkBackToOriginalObject + ); return constantMapCache[val]; } + static PyTypeObject* createAVanillaType() { + PyObjectStealer emptyBases(PyTuple_New(0)); + PyObjectStealer emptyDict(PyDict_New()); + + return (PyTypeObject*)PyObject_CallFunction( + (PyObject*)&PyType_Type, + "sOO", + "AVanillaType", + (PyObject*)emptyBases, + (PyObject*)emptyDict + ); + } + + bool isVanillaClassType(PyTypeObject* o) { + static PyTypeObject* aVanillaType = createAVanillaType(); + return o->tp_new == aVanillaType->tp_new; + } + + static bool isPyObjectGloballyIdentifiable(PyObject* h) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + if (PyObject_HasAttrString(h, "__module__") && PyObject_HasAttrString(h, "__name__")) { + PyObjectStealer moduleName(PyObject_GetAttrString(h, "__module__")); + if (!moduleName) { + PyErr_Clear(); + return false; + } + + PyObjectStealer clsName(PyObject_GetAttrString(h, "__name__")); + if (!clsName) { + PyErr_Clear(); + return false; + } + + if (!PyUnicode_Check(moduleName) || !PyUnicode_Check(clsName)) { + return false; + } + + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, moduleName)); + + if (!moduleObject) { + PyErr_Clear(); + return false; + } + + PyObjectStealer obj(PyObject_GetAttr(moduleObject, clsName)); + + if (!obj) { + PyErr_Clear(); + return false; + } + if ((PyObject*)obj == h) { + return true; + } + } + + return false; + } + void becomeInternalizedOf( PyObject* val, std::unordered_map& constantMapCache, - const std::map<::Type*, ::Type*>& groupMap + const std::map<::Type*, ::Type*>& groupMap, + bool linkBackToOriginalObject ) { - ::Type* t = PyInstance::extractTypeFrom(val); + // we're always the internalized version of this object + if (linkBackToOriginalObject) { + mPyObject = incref(val); + } + + auto internalize = [&](PyObject* o) { + return CompilerVisiblePyObj::internalizePyObj( + o, constantMapCache, groupMap, linkBackToOriginalObject + ); + }; + + ::Type* t = PyInstance::extractTypeFrom(val, true); if (t) { if (groupMap.find(t) != groupMap.end()) { @@ -164,25 +401,221 @@ class CompilerVisiblePyObj { return; } + PyObject* environType = staticPythonInstance("os", "_Environ"); + + if (val->ob_type == (PyTypeObject*)environType) { + mKind = Kind::NamedPyObject; + mName = "_Environ"; + mModuleName = "os"; + return; + } + + + if (PyUnicode_Check(val)) { + mKind = Kind::String; + mStringValue = PyUnicode_AsUTF8(val); + return; + } + if (PyTuple_Check(val)) { mKind = Kind::PyTuple; for (long i = 0; i < PyTuple_Size(val); i++) { - mElements.push_back( - CompilerVisiblePyObj::internalizePyObj( - PyTuple_GetItem(val, i), - constantMapCache, - groupMap - ) - ); + mElements.push_back(internalize(PyTuple_GetItem(val, i))); } return; } + if (PyList_Check(val)) { + mKind = Kind::PyList; + for (long i = 0; i < PyList_Size(val); i++) { + mElements.push_back(internalize(PyList_GetItem(val, i))); + } + return; + } + + if (PySet_Check(val)) { + mKind = Kind::PySet; + iterate(val, [&](PyObject* o) { + mElements.push_back(internalize(o)); + }); + return; + } + + if (PyDict_Check(val)) { + // see if this is a moduledict + PyObject* mname = PyDict_GetItemString(val, "__name__"); + if (mname && PyUnicode_Check(mname)) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + PyObjectStealer moduleObj(PyObject_GetItem(sysModuleModules, mname)); + + if (moduleObj) { + PyObjectStealer moduleObjDict(PyObject_GenericGetDict(moduleObj, nullptr)); + if (!moduleObjDict) { + PyErr_Clear(); + } else + if (moduleObjDict == val) { + mKind = Kind::PyModuleDict; + mName = PyUnicode_AsUTF8(mname); + mNamedElements["module_dict_of"] = internalize(moduleObj); + return; + } + } else { + PyErr_Clear(); + } + } + + // see if this is a vanilla class dict + PyObject* dictAccessor = PyDict_GetItemString(val, "__dict__"); + if (dictAccessor && dictAccessor->ob_type == &PyGetSetDescr_Type) { + PyTypeObject* clsType = PyDescr_TYPE(dictAccessor); + if (clsType) { + if (clsType->tp_dict == val) { + if (isVanillaClassType(clsType)) { + mKind = Kind::PyClassDict; + + mNamedElements["class_dict_of"] = internalize( + (PyObject*)clsType + ); + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (val && PyDict_Next(val, &pos, &key, &value)) { + if (PyUnicode_Check(key) + && PyUnicode_AsUTF8(key) != std::string("__dict__") + && PyUnicode_AsUTF8(key) != std::string("__weakref__") + ) { + mElements.push_back(internalize(value)); + mKeys.push_back(internalize(key)); + } + } + + return; + } + } + } + } + + // this is a vanilla dict + mKind = Kind::PyDict; + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (val && PyDict_Next(val, &pos, &key, &value)) { + mElements.push_back(internalize(value)); + mKeys.push_back(internalize(key)); + } + return; + } + + if (PyCell_Check(val)) { + mKind = Kind::PyCell; + + if (PyCell_Get(val)) { + mNamedElements["cell_contents"] = internalize(PyCell_Get(val)); + } + + return; + } + + if (PyType_Check(val)) { + PyTypeObject* tp = (PyTypeObject*)val; + + if (isVanillaClassType(tp)) { + mKind = Kind::PyClass; + + mName = tp->tp_name; + mModuleName = dictGetStringOrEmpty(tp->tp_dict, "__module__"); + + if (tp->tp_dict) { + mNamedElements["cls_dict"] = internalize(tp->tp_dict); + } + if (tp->tp_bases) { + mNamedElements["cls_bases"] = internalize(tp->tp_bases); + } + + return; + } + } + + if (PyFunction_Check(val)) { + mKind = Kind::PyFunction; + + PyFunctionObject* f = (PyFunctionObject*)val; + + mName = stringOrEmpty(f->func_name); + mModuleName = stringOrEmpty(f->func_module); + + if (f->func_name) { + mNamedElements["func_name"] = internalize(f->func_name); + } + if (f->func_module) { + mNamedElements["func_module"] = internalize(f->func_module); + } + if (f->func_qualname) { + mNamedElements["func_qualname"] = internalize(f->func_qualname); + } + if (PyFunction_GetClosure(val)) { + mNamedElements["func_closure"] = internalize(PyFunction_GetClosure(val)); + } + if (PyFunction_GetCode(val)) { + mNamedElements["func_code"] = internalize(PyFunction_GetCode(val)); + } + if (PyFunction_GetModule(val)) { + mNamedElements["func_module"] = internalize(PyFunction_GetModule(val)); + } + if (PyFunction_GetAnnotations(val)) { + mNamedElements["func_annotations"] = internalize(PyFunction_GetAnnotations(val)); + } + if (PyFunction_GetDefaults(val)) { + mNamedElements["func_defaults"] = internalize(PyFunction_GetDefaults(val)); + } + if (PyFunction_GetKwDefaults(val)) { + mNamedElements["func_kwdefaults"] = internalize(PyFunction_GetKwDefaults(val)); + } + if (PyFunction_GetGlobals(val)) { + mNamedElements["func_globals"] = internalize(PyFunction_GetGlobals(val)); + } + return; + } + + if (PyCode_Check(val)) { + mKind = Kind::PyCodeObject; + + PyCodeObject* co = (PyCodeObject*)val; + + mNamedInts["co_argcount"] = co->co_argcount; + mNamedInts["co_kwonlyargcount"] = co->co_kwonlyargcount; + mNamedInts["co_nlocals"] = co->co_nlocals; + mNamedInts["co_stacksize"] = co->co_stacksize; + mNamedInts["co_firstlineno"] = co->co_firstlineno; + mNamedInts["co_posonlyargcount"] = co->co_posonlyargcount; + mNamedInts["co_flags"] = co->co_flags; + + mNamedElements["co_code"] = internalize(co->co_code); + mNamedElements["co_consts"] = internalize(co->co_consts); + mNamedElements["co_names"] = internalize(co->co_names); + mNamedElements["co_varnames"] = internalize(co->co_varnames); + mNamedElements["co_freevars"] = internalize(co->co_freevars); + mNamedElements["co_cellvars"] = internalize(co->co_cellvars); + mNamedElements["co_name"] = internalize(co->co_name); + mNamedElements["co_filename"] = internalize(co->co_filename); + +# if PY_MINOR_VERSION >= 10 + mNamedElements["co_linetable"] = internalize(co->co_linetable); +# else + mNamedElements["co_lnotab"] = internalize(co->co_lnotab); +# endif + + return; + } + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); if (instanceType) { mKind = Kind::Instance; mInstance = ::Instance::create( - instanceType, + instanceType, ((PyInstance*)val)->dataPtr() ); return; @@ -201,7 +634,7 @@ class CompilerVisiblePyObj { if (PyLong_Check(val)) { mKind = Kind::Instance; - + try { mInstance = Instance::create((int64_t)PyLong_AsLongLong(val)); } @@ -218,14 +651,36 @@ class CompilerVisiblePyObj { return; } + if (PyModule_Check(val)) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + PyObjectStealer name(PyObject_GetAttrString(val, "__name__")); + if (name) { + if (PyUnicode_Check(name)) { + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, name)); + if (moduleObject) { + if (moduleObject == val) { + mKind = Kind::PyModule; + mName = PyUnicode_AsUTF8(name); + return; + } + } else { + PyErr_Clear(); + } + } + } else { + PyErr_Clear(); + } + } + if (PyBytes_Check(val)) { mKind = Kind::Instance; mInstance = Instance::createAndInitialize( BytesType::Make(), [&](instance_ptr i) { BytesType::Make()->constructor( - i, - PyBytes_GET_SIZE(val), + i, + PyBytes_GET_SIZE(val), PyBytes_AsString(val) ); } @@ -233,37 +688,71 @@ class CompilerVisiblePyObj { return; } - if (PyUnicode_Check(val)) { - mKind = Kind::Instance; + if (isVanillaClassType(val->ob_type)) { + mKind = Kind::PyObject; - auto kind = PyUnicode_KIND(val); - assert( - kind == PyUnicode_1BYTE_KIND || - kind == PyUnicode_2BYTE_KIND || - kind == PyUnicode_4BYTE_KIND - ); - int64_t bytesPerCodepoint = - kind == PyUnicode_1BYTE_KIND ? 1 : - kind == PyUnicode_2BYTE_KIND ? 2 : - 4 ; + mNamedElements["inst_type"] = internalize((PyObject*)val->ob_type); + + PyObjectStealer dict(PyObject_GenericGetDict(val, nullptr)); + if (dict) { + mNamedElements["inst_dict"] = internalize(dict); + } + return; + } - int64_t count = PyUnicode_GET_LENGTH(val); + if (val->ob_type == &PyStaticMethod_Type || val->ob_type == &PyClassMethod_Type) { + if (val->ob_type == &PyStaticMethod_Type) { + mKind = Kind::PyStaticMethod; + } else { + mKind = Kind::PyClassMethod; + } + + PyObjectStealer funcObj(PyObject_GetAttrString(val, "__func__")); - const char* data = - kind == PyUnicode_1BYTE_KIND ? (char*)PyUnicode_1BYTE_DATA(val) : - kind == PyUnicode_2BYTE_KIND ? (char*)PyUnicode_2BYTE_DATA(val) : - (char*)PyUnicode_4BYTE_DATA(val); + mNamedElements["meth_func"] = internalize(funcObj); - mInstance = Instance::createAndInitialize( - StringType::Make(), - [&](instance_ptr i) { - StringType::Make()->constructor(i, bytesPerCodepoint, count, data); - } - ); + return; + } + + if (val->ob_type == &PyProperty_Type) { + mKind = Kind::PyProperty; + + PyObjectStealer fget(PyObject_GetAttrString(val, "fget")); + PyObjectStealer fset(PyObject_GetAttrString(val, "fset")); + PyObjectStealer fdel(PyObject_GetAttrString(val, "fdel")); + + mNamedElements["prop_fget"] = internalize(fget); + mNamedElements["prop_fset"] = internalize(fdel); + mNamedElements["prop_fdel"] = internalize(fset); return; } + if (val->ob_type == &PyMethod_Type) { + mKind = Kind::PyBoundMethod; + + PyObjectStealer fself(PyObject_GetAttrString(val, "__self__")); + PyObjectStealer ffunc(PyObject_GetAttrString(val, "__func__")); + + mNamedElements["meth_self"] = internalize(fself); + mNamedElements["meth_func"] = internalize(ffunc); + + return; + } + + if (isPyObjectGloballyIdentifiable(val)) { + mKind = Kind::NamedPyObject; + + // no checks are necessary because isPyObjectGloballyIdentifiable + // confirms that this is OK. + PyObjectStealer moduleName(PyObject_GetAttrString(val, "__module__")); + PyObjectStealer clsName(PyObject_GetAttrString(val, "__name__")); + + mModuleName = PyUnicode_AsUTF8(moduleName); + mName = PyUnicode_AsUTF8(clsName); + return; + } + mKind = Kind::ArbitraryPyObject; mPyObject = incref(val); } @@ -280,6 +769,18 @@ class CompilerVisiblePyObj { return mElements; } + const std::vector& keys() const { + return mKeys; + } + + const std::map& namedElements() const { + return mNamedElements; + } + + const std::map& namedInts() const { + return mNamedInts; + } + ::Type* getType() const { return mType; } @@ -288,6 +789,18 @@ class CompilerVisiblePyObj { return mInstance; } + const std::string& getStringValue() const { + return mStringValue; + } + + const std::string& getName() const { + return mName; + } + + const std::string& getModuleName() const { + return mModuleName; + } + template void _visitReferencedTypes(const visitor_type& v) { if (mKind == Kind::Type) { @@ -326,46 +839,264 @@ class CompilerVisiblePyObj { } // get the python object representation of this object, which isn't guaranteed - // to exist and may need to be constructed on demand. + // to exist and may need to be constructed on demand. this will do a pass over + // all reachable objects, building skeletons, and then performs a second pass where + // we fill items out once we have the skeletons in place. PyObject* getPyObj() { - if (mKind == Kind::Type) { - return (PyObject*)PyInstance::typeObj(mType); - } - - if (mKind == Kind::ArbitraryPyObject) { + if (mPyObject) { return mPyObject; } - if (mKind == Kind::Instance) { - if (!mPyObject) { - mPyObject = PyInstance::extractPythonObject(mInstance); - } + std::unordered_set needsResolution; + + PyObject* res = getPyObj(needsResolution); + + for (auto n: needsResolution) { + n->finalizeGetPyObj(); + } - return mPyObject; + // do a second pass patching up classes. They don't inherit things + // from their dict correctly + for (auto n: needsResolution) { + n->finalizeGetPyObj2(); } - throw std::runtime_error("Can't make a python object representation for this pyobj"); + return res; } - std::string toString() { - if (mKind == Kind::Type) { - return "CompilerVisiblePyObj.Type(" + mType->name() + ")"; + void finalizeGetPyObj2() { + if (mKind == Kind::PyClass) { + if (mNamedElements.find("cls_dict") == mNamedElements.end()) { + throw std::runtime_error("Corrupt PyClass - no cls_dict"); + } + + CompilerVisiblePyObj* clsDictPyObj = mNamedElements["cls_dict"]; + + for (long k = 0; k < clsDictPyObj->elements().size() && k < clsDictPyObj->keys().size(); k++) { + if (clsDictPyObj->keys()[k]->isString()) { + PyObject_SetAttrString( + mPyObject, + clsDictPyObj->keys()[k]->getStringValue().c_str(), + clsDictPyObj->elements()[k]->getPyObj() + ); + } + } } + } - if (mKind == Kind::Instance) { - return "CompilerVisiblePyObj.Instance(" + mInstance.type()->name() + ")"; + // we're a skeleton - finish ourselves out + void finalizeGetPyObj() { + if (mKind == Kind::PyDict || mKind == Kind::PyClassDict) { + for (long k = 0; k < mElements.size() && k < mKeys.size(); k++) { + PyDict_SetItem( + mPyObject, + mKeys[k]->getPyObj(), + mElements[k]->getPyObj() + ); + } + } else if (mKind == Kind::PyCell) { + auto it = mNamedElements.find("cell_contents"); + if (it != mNamedElements.end()) { + PyCell_Set( + mPyObject, + it->second->getPyObj() + ); + } } + } - if (mKind == Kind::PyTuple) { - return "CompilerVisiblePyObj.PyTuple()"; + PyObject* getPyObj(std::unordered_set& needsResolution) { + if (!mPyObject) { + if (mKind == Kind::Type) { + mPyObject = (PyObject*)PyInstance::typeObj(mType); + } else if (mKind == Kind::String) { + mPyObject = PyUnicode_FromString(mStringValue.c_str()); + } else if (mKind == Kind::Instance) { + mPyObject = PyInstance::extractPythonObject(mInstance); + } else if (mKind == Kind::PyDict || mKind == Kind::PyClassDict) { + mPyObject = PyDict_New(); + needsResolution.insert(this); + } else if (mKind == Kind::PyList) { + mPyObject = PyList_New(0); + + for (long k = 0; k < mElements.size(); k++) { + PyList_Append(mPyObject, mElements[k]->getPyObj(needsResolution)); + } + } else if (mKind == Kind::PyTuple) { + mPyObject = PyTuple_New(mElements.size()); + + // first initialize it in case we throw somehow + for (long k = 0; k < mElements.size(); k++) { + PyTuple_SetItem(mPyObject, k, incref(Py_None)); + } + + for (long k = 0; k < mElements.size(); k++) { + PyTuple_SetItem(mPyObject, k, incref(mElements[k]->getPyObj(needsResolution))); + } + } else if (mKind == Kind::PySet) { + mPyObject = PySet_New(nullptr); + + for (long k = 0; k < mElements.size(); k++) { + PySet_Add(mPyObject, incref(mElements[k]->getPyObj(needsResolution))); + } + } else if (mKind == Kind::PyClass) { + needsResolution.insert(this); + + PyObjectStealer argTup(PyTuple_New(3)); + PyTuple_SetItem(argTup, 0, PyUnicode_FromString(mName.c_str())); + PyTuple_SetItem(argTup, 1, incref(getNamedElementPyobj("cls_bases", needsResolution))); + PyTuple_SetItem(argTup, 2, incref(getNamedElementPyobj("cls_dict", needsResolution))); + + mPyObject = PyType_Type.tp_new(&PyType_Type, argTup, nullptr); + + if (!mPyObject) { + throw PythonExceptionSet(); + } + } else if (mKind == Kind::PyModule) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + PyObjectStealer nameAsStr(PyUnicode_FromString(mName.c_str())); + PyObjectStealer moduleObject( + PyObject_GetItem(sysModuleModules, nameAsStr) + ); + + if (!moduleObject) { + PyErr_Clear(); + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + mName + " is not loaded!"); + } + + mPyObject = incref(moduleObject); + } else if (mKind == Kind::PyModuleDict) { + mPyObject = PyObject_GenericGetDict(getNamedElementPyobj("module_dict_of", needsResolution), nullptr); + if (!mPyObject) { + throw PythonExceptionSet(); + } + } else if (mKind == Kind::PyCell) { + mPyObject = PyCell_New(nullptr); + needsResolution.insert(this); + } else if (mKind == Kind::PyObject) { + PyObject* t = getNamedElementPyobj("inst_type", needsResolution); + PyObject* d = getNamedElementPyobj("inst_dict", needsResolution); + + static PyObject* emptyTuple = PyTuple_Pack(0); + + mPyObject = ((PyTypeObject*)t)->tp_new( + ((PyTypeObject*)t), + emptyTuple, + nullptr + ); + + if (PyObject_GenericSetDict(mPyObject, d, nullptr)) { + throw PythonExceptionSet(); + } + } else if (mKind == Kind::PyFunction) { + mPyObject = PyFunction_New( + getNamedElementPyobj("func_code", needsResolution), + getNamedElementPyobj("func_globals", needsResolution) + ); + + if (mNamedElements.find("func_closure") != mNamedElements.end()) { + PyFunction_SetClosure( + mPyObject, + getNamedElementPyobj("func_closure", needsResolution) + ); + } + + if (mNamedElements.find("func_annotations") != mNamedElements.end()) { + PyFunction_SetAnnotations( + mPyObject, + getNamedElementPyobj("func_annotations", needsResolution) + ); + } + + if (mNamedElements.find("func_defaults") != mNamedElements.end()) { + PyFunction_SetAnnotations( + mPyObject, + getNamedElementPyobj("func_defaults", needsResolution) + ); + } + + if (mNamedElements.find("func_qualname") != mNamedElements.end()) { + PyObject_SetAttrString( + mPyObject, + "__qualname__", + getNamedElementPyobj("func_qualname", needsResolution) + ); + } + + if (mNamedElements.find("func_kwdefaults") != mNamedElements.end()) { + PyObject_SetAttrString( + mPyObject, + "__kwdefaults__", + getNamedElementPyobj("func_kwdefaults", needsResolution) + ); + } + + if (mNamedElements.find("func_name") != mNamedElements.end()) { + PyObject_SetAttrString( + mPyObject, + "__name__", + getNamedElementPyobj("func_name", needsResolution) + ); + } + } else if (mKind == Kind::PyCodeObject) { +#if PY_MINOR_VERSION < 8 + mPyObject = (PyObject*)PyCode_New( +#else + mPyObject = (PyObject*)PyCode_NewWithPosOnlyArgs( +#endif + mNamedInts["co_argcount"], +#if PY_MINOR_VERSION >= 8 + mNamedInts["co_posonlyargcount"], +#endif + mNamedInts["co_kwonlyargcount"], + mNamedInts["co_nlocals"], + mNamedInts["co_stacksize"], + mNamedInts["co_flags"], + getNamedElementPyobj("co_code", needsResolution), + getNamedElementPyobj("co_consts", needsResolution), + getNamedElementPyobj("co_names", needsResolution), + getNamedElementPyobj("co_varnames", needsResolution), + getNamedElementPyobj("co_freevars", needsResolution), + getNamedElementPyobj("co_cellvars", needsResolution), + getNamedElementPyobj("co_filename", needsResolution), + getNamedElementPyobj("co_name", needsResolution), + mNamedInts["co_firstlineno"], + getNamedElementPyobj( +#if PY_MINOR_VERSION >= 10 + "co_linetable", +#else + "co_lnotab", +#endif + needsResolution + ) + ); + } else { + throw std::runtime_error( + "Can't make a python object representation for a CVPO of kind " + kindAsString() + ); + } } - if (mKind == Kind::ArbitraryPyObject) { - return std::string("CompilerVisiblePyObj.ArbitraryPyObject(type=") - + mPyObject->ob_type->tp_name + ")"; + return mPyObject; + } + + PyObject* getNamedElementPyobj( + std::string name, + std::unordered_set& needsResolution + ) { + auto it = mNamedElements.find(name); + if (it == mNamedElements.end()) { + throw std::runtime_error( + "Corrupt CompilerVisiblePyObj." + kindAsString() + ". missing " + name + ); } - throw std::runtime_error("Unknown CompilerVisiblePyObj Kind."); + return it->second->getPyObj(needsResolution); + } + + std::string toString() const { + return "CompilerVisiblePyObj." + kindAsString() + "()"; } template @@ -418,10 +1149,19 @@ class CompilerVisiblePyObj { ::Instance i; uint32_t id = -1; PyObjectHolder pyobj; + CompilerVisiblePyObj* result = nullptr; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { id = buffer.readUnsignedVarint(); + + void* ptr = buffer.lookupCachedPointer(id); + if (ptr) { + result = (CompilerVisiblePyObj*)result; + } else { + result = new CompilerVisiblePyObj(); + buffer.addCachedPointer(id, result, nullptr); + } } else if (fieldNumber == 1) { kind = buffer.readUnsignedVarint(); @@ -442,46 +1182,59 @@ class CompilerVisiblePyObj { } }); - void* ptr = buffer.lookupCachedPointer(id); - if (ptr) { - return (CompilerVisiblePyObj*)ptr; + if (!result) { + throw std::runtime_error("corrupt CompilerVisiblePyObj - no memo found"); } - if (kind == (int)Kind::Type) { - if (!type) { - throw std::runtime_error("Corrupt CompilerVisiblePyObj::Type"); - } - return CompilerVisiblePyObj::Type(type); + if (kind == -1) { + throw std::runtime_error("corrupt CompilerVisiblePyObj - invalid kind"); } - if (kind == (int)Kind::Instance) { - return CompilerVisiblePyObj::Instance(i); - } + result->mKind = Kind(kind); + result->mType = type; + result->mInstance = i; + result->mPyObject = incref(pyobj); + result->mElements = vec; - if (kind == (int)Kind::PyTuple) { - return CompilerVisiblePyObj::PyTuple(vec); - } + result->validateAfterDeserialization(); - if (kind == (int)Kind::Uninitialized) { - return new CompilerVisiblePyObj(); - } + return result; + } - if (kind == (int)Kind::ArbitraryPyObject) { - return CompilerVisiblePyObj::ArbitraryPyObject(pyobj); +private: + // ensure we won't crash if we interact with this object. + void validateAfterDeserialization() { + if (mKind == Kind::Type) { + if (!mType) { + throw std::runtime_error("Corrupt CVPO: no Type"); + } } - throw std::runtime_error("Corrupt CompilerVisiblePyObj - invalid kind"); + if (mKind == Kind::ArbitraryPyObject) { + if (!mPyObject) { + throw std::runtime_error("Corrupt CVPO: no PyObject"); + } + } } - -private: Kind mKind; ::Type* mType; ::Instance mInstance; // if we are an ArbitraryPythonObject this is always populated - // otherwise, it will be a cache for a constructed instance + // otherwise, it will be a cache for a constructed instance of the object PyObject* mPyObject; + std::map mNamedElements; + std::map mNamedInts; + + // if we're a tuple, list, or dict std::vector mElements; + + // if we're a PyDict + std::vector mKeys; + + std::string mStringValue; + std::string mModuleName; + std::string mName; }; diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 2dbcd1f69..4e7a79880 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -160,11 +160,26 @@ class FunctionGlobal { } Type* getValueAsType() { - PyObject* obj = getValueAsPyobj(); - if (obj) { - return PyInstance::extractTypeFrom(obj); + if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { + PyObject* ref = extractGlobalRefFromDictOrCell(); + if (!ref) { + return nullptr; + } + return PyInstance::extractTypeFrom(ref); } - return nullptr; + + if (isUnbound()) { + return nullptr; + } + + if (isConstant()) { + if (mConstant->isType()) { + return mConstant->getType(); + } + return nullptr; + } + + throw std::runtime_error("Unknown global kind."); } PyObject* getValueAsPyobj() { @@ -320,11 +335,11 @@ class FunctionGlobal { auto it = groupMap.find(t); if (it != groupMap.end()) { return FunctionGlobal::Constant( - CompilerVisiblePyObj::Type(it->second) + CompilerVisiblePyObj::ForType(it->second) ); } else { return FunctionGlobal::Constant( - CompilerVisiblePyObj::Type(t) + CompilerVisiblePyObj::ForType(t) ); } } @@ -356,7 +371,7 @@ class FunctionGlobal { if (it != groupMap.end()) { // we're actually a constant! return FunctionGlobal::Constant( - CompilerVisiblePyObj::Type(it->second) + CompilerVisiblePyObj::ForType(it->second) ); } diff --git a/typed_python/PyCompilerVisiblePyObj.cpp b/typed_python/PyCompilerVisiblePyObj.cpp index aeb62c819..9cc76572e 100644 --- a/typed_python/PyCompilerVisiblePyObj.cpp +++ b/typed_python/PyCompilerVisiblePyObj.cpp @@ -22,12 +22,41 @@ PyDoc_STRVAR(PyCompilerVisiblePyObj_doc, ); PyMethodDef PyCompilerVisiblePyObj_methods[] = { + {"create", (PyCFunction)PyCompilerVisiblePyObj::create, METH_VARARGS | METH_KEYWORDS | METH_STATIC, NULL}, {NULL} /* Sentinel */ }; + +/* static */ +PyObject* PyCompilerVisiblePyObj::create(PyObject* self, PyObject* args, PyObject* kwargs) { + return translateExceptionToPyObject([&]() { + PyObject* instance; + int linkBack = 1; + static const char *kwlist[] = {"instance", "linkBackToOriginalObject", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|p", (char**)kwlist, &instance, &linkBack)) { + throw PythonExceptionSet(); + } + + std::unordered_map constantMapCache; + std::map<::Type*, ::Type*> groupMap; + + CompilerVisiblePyObj* object = CompilerVisiblePyObj::internalizePyObj( + instance, + constantMapCache, + groupMap, + linkBack + ); + + return PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(object); + }); +} + /* static */ void PyCompilerVisiblePyObj::dealloc(PyCompilerVisiblePyObj *self) { + decref(self->mElements); + decref(self->mKeys); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -41,6 +70,9 @@ PyObject* PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(CompilerVisiblePyObj PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)PyType_CompilerVisiblePyObj.tp_alloc(&PyType_CompilerVisiblePyObj, 0); self->mPyobj = g; + self->mElements = nullptr; + self->mKeys = nullptr; + self->mByKey = nullptr; memo[g] = incref((PyObject*)self); @@ -56,6 +88,9 @@ PyObject* PyCompilerVisiblePyObj::new_(PyTypeObject *type, PyObject *args, PyObj if (self != NULL) { self->mPyobj = nullptr; + self->mElements = nullptr; + self->mKeys = nullptr; + self->mByKey = nullptr; } return (PyObject*)self; @@ -73,48 +108,98 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN CompilerVisiblePyObj* obj = pyCVPO->mPyobj; - if (attr == "kind") { - if (obj->isUninitialized()) { - return PyUnicode_FromString("Uninitialized"); - } + auto it = obj->namedElements().find(attr); + if (it != obj->namedElements().end()) { + return PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(it->second); + } - if (obj->isType()) { - return PyUnicode_FromString("Type"); - } + auto it2 = obj->namedInts().find(attr); + if (it2 != obj->namedInts().end()) { + return PyLong_FromLong(it2->second); + } - if (obj->isInstance()) { - return PyUnicode_FromString("Instance"); + if (attr == "pyobj") { + PyObject* o = obj->getPyObj(); + if (!o) { + throw std::runtime_error("CVPO of kind " + obj->kindAsString() + " has no pyobj"); } + return incref(o); + } - if (obj->isPyTuple()) { - return PyUnicode_FromString("PyTuple"); - } + if (attr == "kind") { + return PyUnicode_FromString(obj->kindAsString().c_str()); + } - if (obj->isArbitraryPyObject()) { - return PyUnicode_FromString("ArbitraryPyObject"); - } + if (attr == "type" && obj->getType()) { + return incref((PyObject*)PyInstance::typeObj(obj->getType())); + } - throw std::runtime_error("Unknown CompilerVisiblePyObj Kind"); + if (attr == "instance" && obj->isInstance()) { + return PyInstance::fromInstance(obj->getInstance()); + } + + if (attr == "stringValue" && obj->isString()) { + return PyUnicode_FromString(obj->getStringValue().c_str()); + } + + if (attr == "name") { + return PyUnicode_FromString(obj->getName().c_str()); } - if (attr == "type" && obj->isType()) { - Type* t = obj->getType(); + if (attr == "moduleName") { + return PyUnicode_FromString(obj->getModuleName().c_str()); + } - if (!t) { - throw std::runtime_error("Somehow we don't have a type"); + if (attr == "elements") { + if (!pyCVPO->mElements) { + pyCVPO->mElements = PyList_New(0); + for (auto p: obj->elements()) { + PyList_Append(pyCVPO->mElements, PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(p)); + } } - return incref((PyObject*)PyInstance::typeObj(t)); + return incref(pyCVPO->mElements); } - if (attr == "instance" && obj->isInstance()) { - return PyInstance::fromInstance(obj->getInstance()); + if (attr == "keys") { + if (!pyCVPO->mKeys) { + pyCVPO->mKeys = PyList_New(0); + for (auto p: obj->keys()) { + PyList_Append(pyCVPO->mKeys, PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(p)); + } + } + + return incref(pyCVPO->mKeys); + } + + if (attr == "byKey") { + if (!pyCVPO->mByKey) { + pyCVPO->mByKey = PyDict_New(); + for (long k = 0; k < obj->keys().size(); k++) { + PyDict_SetItem( + pyCVPO->mByKey, + obj->keys()[k]->getPyObj(), + PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(obj->elements()[k]) + ); + } + } + + return incref(pyCVPO->mByKey); } return PyObject_GenericGetAttr(selfObj, attrName); }); } +PyObject* PyCompilerVisiblePyObj::tp_repr(PyObject *selfObj) { + PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)selfObj; + + return translateExceptionToPyObject([&]() { + return PyUnicode_FromString(self->mPyobj->toString().c_str()); + }); +} + + /* static */ int PyCompilerVisiblePyObj::init(PyCompilerVisiblePyObj *self, PyObject *args, PyObject *kwargs) { @@ -122,6 +207,7 @@ int PyCompilerVisiblePyObj::init(PyCompilerVisiblePyObj *self, PyObject *args, P return -1; } + PyTypeObject PyType_CompilerVisiblePyObj = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "CompilerVisiblePyObj", @@ -136,7 +222,7 @@ PyTypeObject PyType_CompilerVisiblePyObj = { .tp_getattr = 0, .tp_setattr = 0, .tp_as_async = 0, - .tp_repr = 0, + .tp_repr = PyCompilerVisiblePyObj::tp_repr, .tp_as_number = 0, .tp_as_sequence = 0, .tp_as_mapping = 0, diff --git a/typed_python/PyCompilerVisiblePyObj.hpp b/typed_python/PyCompilerVisiblePyObj.hpp index 63851f60d..653cbfc75 100644 --- a/typed_python/PyCompilerVisiblePyObj.hpp +++ b/typed_python/PyCompilerVisiblePyObj.hpp @@ -23,6 +23,13 @@ class PyCompilerVisiblePyObj { PyObject_HEAD CompilerVisiblePyObj* mPyobj; + PyObject* mElements; + PyObject* mKeys; + PyObject* mByKey; + + static PyObject* tp_repr(PyObject *selfObj); + + static PyObject* create(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* tp_getattro(PyObject* selfObj, PyObject* attr); diff --git a/typed_python/compiler_visible_py_obj_test.py b/typed_python/compiler_visible_py_obj_test.py new file mode 100644 index 000000000..dd52bbb6c --- /dev/null +++ b/typed_python/compiler_visible_py_obj_test.py @@ -0,0 +1,188 @@ +import numpy + +from typed_python import ListOf +from typed_python._types import CompilerVisiblePyObj + + +def test_make_cvpo_basic(): + assert CompilerVisiblePyObj.create(int).kind == 'Type' + + assert CompilerVisiblePyObj.create((1, 2, 3)).kind == 'PyTuple' + assert CompilerVisiblePyObj.create((1, 2, 3)).elements[0].kind == 'Instance' + + assert CompilerVisiblePyObj.create({1, 2, 3}).kind == 'PySet' + assert CompilerVisiblePyObj.create({1, 2, 3}).elements[0].kind == 'Instance' + + assert CompilerVisiblePyObj.create([1, 2, 3]).kind == 'PyList' + assert CompilerVisiblePyObj.create([1, 2, 3]).elements[0].kind == 'Instance' + + assert CompilerVisiblePyObj.create({1:2}).kind == 'PyDict' + assert CompilerVisiblePyObj.create({'1':2}).elements[0].kind == 'Instance' + assert CompilerVisiblePyObj.create({'1':2}).keys[0].kind == 'String' + + assert CompilerVisiblePyObj.create(1).kind == 'Instance' + assert CompilerVisiblePyObj.create(1).instance == 1 + + assert CompilerVisiblePyObj.create('1').kind == 'String' + assert CompilerVisiblePyObj.create('1').stringValue == '1' + + assert CompilerVisiblePyObj.create(ListOf(int)((1, 2, 3))).kind == 'Instance' + assert CompilerVisiblePyObj.create(ListOf(int)((1, 2, 3))).instance[1] == 2 + + +def test_cvpo_function(): + y = 10 + + def f(x: int, z=20): + return 10 + y + + cvpo = CompilerVisiblePyObj.create(f) + + assert cvpo.kind == 'PyFunction' + assert cvpo.name == 'f' + assert cvpo.moduleName == 'typed_python.compiler_visible_py_obj_test' + assert cvpo.func_name.kind == 'String' + assert cvpo.func_module.kind == 'String' + assert cvpo.func_closure.kind == 'PyTuple' + assert cvpo.func_closure.elements[0].kind == 'PyCell' + assert cvpo.func_closure.elements[0].cell_contents.pyobj == 10 + assert cvpo.func_globals.kind == "PyModuleDict" + assert cvpo.func_globals.name == 'typed_python.compiler_visible_py_obj_test' + assert cvpo.func_annotations.pyobj['x'] is int + assert cvpo.func_defaults.pyobj == (20,) + assert not hasattr(cvpo, 'func_kwdefaults') + + assert cvpo.func_code.kind == 'PyCodeObject' + assert cvpo.func_code.co_argcount == 2 + assert cvpo.func_code.co_kwonlyargcount == 0 + assert cvpo.func_code.co_flags == f.__code__.co_flags + assert cvpo.func_code.co_posonlyargcount == f.__code__.co_posonlyargcount + assert cvpo.func_code.co_nlocals == f.__code__.co_nlocals + assert cvpo.func_code.co_stacksize == f.__code__.co_stacksize + assert cvpo.func_code.co_firstlineno == f.__code__.co_firstlineno + assert cvpo.func_code.co_code.pyobj == f.__code__.co_code + assert cvpo.func_code.co_consts.pyobj == f.__code__.co_consts + assert cvpo.func_code.co_names.pyobj == f.__code__.co_names + assert cvpo.func_code.co_varnames.pyobj == f.__code__.co_varnames + assert cvpo.func_code.co_freevars.pyobj == f.__code__.co_freevars + assert cvpo.func_code.co_cellvars.pyobj == f.__code__.co_cellvars + assert cvpo.func_code.co_name.pyobj == f.__code__.co_name + assert cvpo.func_code.co_filename.pyobj == f.__code__.co_filename + + +def test_cvpo_numpy_internals(): + assert CompilerVisiblePyObj.create(numpy.array).kind == 'NamedPyObject' + + # in theory, we could do better than this by using the reduce pathway. + # however, that would involve calling into arbitrary python code during walk time + # which is dangerous because it can (a) modify objects during the walk and + # (b) it can release the GIL allowing other threads to run (who might then + # modify objects) all of which make it impossible to get a good trace of the + # graph. + assert CompilerVisiblePyObj.create(numpy.sin).kind == 'ArbitraryPyObject' + assert CompilerVisiblePyObj.create(numpy.array([1])).kind == 'ArbitraryPyObject' + + +def test_cvpo_builtins(): + assert CompilerVisiblePyObj.create(__builtins__).kind == 'PyModuleDict' + assert CompilerVisiblePyObj.create(__builtins__).module_dict_of.kind == 'PyModule' + assert CompilerVisiblePyObj.create(set).kind == 'NamedPyObject' + assert CompilerVisiblePyObj.create(print).kind == 'NamedPyObject' + assert CompilerVisiblePyObj.create(len).kind == 'NamedPyObject' + assert CompilerVisiblePyObj.create(range).kind == 'NamedPyObject' + + +def test_cvpo_class(): + class C: + def f(self): + return gInst + + def fMeth(self): + return gMeth + + @property + def aProp(self): + return 20 + + class G(C): + def __init__(self): + self.y = 23 + + @staticmethod + def staticMeth(): + return 10 + + @classmethod + def clsMeth(cls): + return 10 + + gInst = G() + gMeth = gInst.f + + cvpo = CompilerVisiblePyObj.create(C) + + assert cvpo.kind == 'PyClass' + assert cvpo.name == 'C' + assert cvpo.moduleName == 'typed_python.compiler_visible_py_obj_test' + assert cvpo.cls_dict.kind == 'PyClassDict' + + assert cvpo.cls_dict.byKey['aProp'].kind == 'PyProperty' + assert cvpo.cls_dict.byKey['aProp'].prop_fget.kind == 'PyFunction' + + assert cvpo.cls_dict.byKey['f'].kind == 'PyFunction' + + gMethPO = cvpo.cls_dict.byKey['fMeth'].func_closure.elements[0].cell_contents + assert gMethPO.kind == 'PyBoundMethod' + assert gMethPO.meth_self.pyobj is gInst + assert gMethPO.meth_func.kind == 'PyFunction' + + gInstPO = cvpo.cls_dict.byKey['f'].func_closure.elements[0].cell_contents + assert gInstPO.pyobj is gInst + assert gInstPO.inst_dict.kind == 'PyDict' + gInstPO.inst_dict.byKey['y'].kind == 'Instance' + + GPO = gInstPO.inst_type + assert GPO.kind == 'PyClass' + assert GPO.cls_bases.pyobj == (C,) + + # we can't actually tell that this is a class dict because + # its not the base class + assert GPO.cls_dict.kind == 'PyDict' + + assert GPO.cls_dict.byKey['staticMeth'].kind == 'PyStaticMethod' + assert GPO.cls_dict.byKey['staticMeth'].meth_func.kind == 'PyFunction' + + assert GPO.cls_dict.byKey['clsMeth'].kind == 'PyClassMethod' + assert GPO.cls_dict.byKey['clsMeth'].meth_func.kind == 'PyFunction' + + +def test_cvpo_rehydration(): + assert CompilerVisiblePyObj.create(1, False).pyobj == 1 + assert CompilerVisiblePyObj.create('1', False).pyobj == '1' + assert CompilerVisiblePyObj.create(b'1', False).pyobj == b'1' + assert CompilerVisiblePyObj.create([1, 2, 'hi'], False).pyobj == [1, 2, 'hi'] + assert CompilerVisiblePyObj.create((1, 2, 'hi'), False).pyobj == (1, 2, 'hi') + assert CompilerVisiblePyObj.create({1, 2}, False).pyobj == {1, 2} + assert CompilerVisiblePyObj.create({1: 2,'3': 4}, False).pyobj == {1: 2, '3': 4} + + def f(): + return 1 + + assert CompilerVisiblePyObj.create(f, False).pyobj() == 1 + + class C: + def f(self): + return cInst + + cInst = C() + + C2cvpo = CompilerVisiblePyObj.create(C, False) + C2 = C2cvpo.pyobj + + assert C2 is not C + assert C2.__name__ == C.__name__ + assert C2.__module__ == C.__module__ + assert C2.f is not C.f + c2Inst = C2cvpo.cls_dict.byKey['f'].func_closure.elements[0].cell_contents.pyobj + + assert C2().f() is c2Inst From 7b641c144c91310d327d93ebf67af9cee0191b34 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 27 Jul 2023 22:25:55 +0000 Subject: [PATCH 61/83] [wip] --- typed_python/CompilerVisiblePyObj.hpp | 139 +++++++++++++++---- typed_python/compiler_visible_py_obj_test.py | 16 ++- 2 files changed, 122 insertions(+), 33 deletions(-) diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index 43e01679b..c72691739 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -310,7 +310,7 @@ class CompilerVisiblePyObj { static PyTypeObject* createAVanillaType() { PyObjectStealer emptyBases(PyTuple_New(0)); PyObjectStealer emptyDict(PyDict_New()); - + return (PyTypeObject*)PyObject_CallFunction( (PyObject*)&PyType_Type, "sOO", @@ -472,7 +472,7 @@ class CompilerVisiblePyObj { if (clsType->tp_dict == val) { if (isVanillaClassType(clsType)) { mKind = Kind::PyClassDict; - + mNamedElements["class_dict_of"] = internalize( (PyObject*)clsType ); @@ -481,7 +481,7 @@ class CompilerVisiblePyObj { Py_ssize_t pos = 0; while (val && PyDict_Next(val, &pos, &key, &value)) { - if (PyUnicode_Check(key) + if (PyUnicode_Check(key) && PyUnicode_AsUTF8(key) != std::string("__dict__") && PyUnicode_AsUTF8(key) != std::string("__weakref__") ) { @@ -706,7 +706,7 @@ class CompilerVisiblePyObj { } else { mKind = Kind::PyClassMethod; } - + PyObjectStealer funcObj(PyObject_GetAttrString(val, "__func__")); mNamedElements["meth_func"] = internalize(funcObj); @@ -716,21 +716,34 @@ class CompilerVisiblePyObj { if (val->ob_type == &PyProperty_Type) { mKind = Kind::PyProperty; - - PyObjectStealer fget(PyObject_GetAttrString(val, "fget")); - PyObjectStealer fset(PyObject_GetAttrString(val, "fset")); - PyObjectStealer fdel(PyObject_GetAttrString(val, "fdel")); - mNamedElements["prop_fget"] = internalize(fget); - mNamedElements["prop_fset"] = internalize(fdel); - mNamedElements["prop_fdel"] = internalize(fset); + JustLikeAPropertyObject* prop = (JustLikeAPropertyObject*)mPyObject; + + if (prop->prop_get) { + mNamedElements["prop_get"] = internalize(prop->prop_get); + } + if (prop->prop_set) { + mNamedElements["prop_set"] = internalize(prop->prop_set); + } + if (prop->prop_del) { + mNamedElements["prop_del"] = internalize(prop->prop_del); + } + if (prop->prop_doc) { + mNamedElements["prop_doc"] = internalize(prop->prop_doc); + } + + #if PY_MINOR_VERSION >= 10 + if (prop->prop_name) { + mNamedElements["prop_name"] = internalize(prop->prop_name); + } + #endif return; } if (val->ob_type == &PyMethod_Type) { mKind = Kind::PyBoundMethod; - + PyObjectStealer fself(PyObject_GetAttrString(val, "__self__")); PyObjectStealer ffunc(PyObject_GetAttrString(val, "__func__")); @@ -848,9 +861,9 @@ class CompilerVisiblePyObj { } std::unordered_set needsResolution; - + PyObject* res = getPyObj(needsResolution); - + for (auto n: needsResolution) { n->finalizeGetPyObj(); } @@ -889,7 +902,7 @@ class CompilerVisiblePyObj { if (mKind == Kind::PyDict || mKind == Kind::PyClassDict) { for (long k = 0; k < mElements.size() && k < mKeys.size(); k++) { PyDict_SetItem( - mPyObject, + mPyObject, mKeys[k]->getPyObj(), mElements[k]->getPyObj() ); @@ -906,13 +919,37 @@ class CompilerVisiblePyObj { } PyObject* getPyObj(std::unordered_set& needsResolution) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + if (!mPyObject) { - if (mKind == Kind::Type) { + if (mKind == Kind::ArbitraryPyObject) { + throw std::runtime_error("Corrupt CompilerVisiblePyObj.ArbitraryPyObject: missing mPyObject"); + } else if (mKind == Kind::Type) { mPyObject = (PyObject*)PyInstance::typeObj(mType); } else if (mKind == Kind::String) { mPyObject = PyUnicode_FromString(mStringValue.c_str()); } else if (mKind == Kind::Instance) { mPyObject = PyInstance::extractPythonObject(mInstance); + } else if (mKind == Kind::NamedPyObject) { + PyObjectStealer nameAsStr(PyUnicode_FromString(mModuleName.c_str())); + PyObjectStealer moduleObject( + PyObject_GetItem(sysModuleModules, nameAsStr) + ); + + if (!moduleObject) { + PyErr_Clear(); + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + mModuleName + " is not loaded!"); + } + PyObjectStealer inst(PyObject_GetAttrString(moduleObject, mName.c_str())); + if (!inst) { + PyErr_Clear(); + + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + mModuleName + " is missing member " + mName); + } + + mPyObject = incref(inst); } else if (mKind == Kind::PyDict || mKind == Kind::PyClassDict) { mPyObject = PyDict_New(); needsResolution.insert(this); @@ -946,19 +983,18 @@ class CompilerVisiblePyObj { PyTuple_SetItem(argTup, 0, PyUnicode_FromString(mName.c_str())); PyTuple_SetItem(argTup, 1, incref(getNamedElementPyobj("cls_bases", needsResolution))); PyTuple_SetItem(argTup, 2, incref(getNamedElementPyobj("cls_dict", needsResolution))); - + mPyObject = PyType_Type.tp_new(&PyType_Type, argTup, nullptr); if (!mPyObject) { throw PythonExceptionSet(); } } else if (mKind == Kind::PyModule) { - PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); PyObjectStealer nameAsStr(PyUnicode_FromString(mName.c_str())); PyObjectStealer moduleObject( PyObject_GetItem(sysModuleModules, nameAsStr) ); - + if (!moduleObject) { PyErr_Clear(); // TODO: should we be importing here? that seems dangerous... @@ -998,21 +1034,21 @@ class CompilerVisiblePyObj { if (mNamedElements.find("func_closure") != mNamedElements.end()) { PyFunction_SetClosure( mPyObject, - getNamedElementPyobj("func_closure", needsResolution) + getNamedElementPyobj("func_closure", needsResolution) ); } if (mNamedElements.find("func_annotations") != mNamedElements.end()) { PyFunction_SetAnnotations( mPyObject, - getNamedElementPyobj("func_annotations", needsResolution) + getNamedElementPyobj("func_annotations", needsResolution) ); } if (mNamedElements.find("func_defaults") != mNamedElements.end()) { PyFunction_SetAnnotations( mPyObject, - getNamedElementPyobj("func_defaults", needsResolution) + getNamedElementPyobj("func_defaults", needsResolution) ); } @@ -1020,7 +1056,7 @@ class CompilerVisiblePyObj { PyObject_SetAttrString( mPyObject, "__qualname__", - getNamedElementPyobj("func_qualname", needsResolution) + getNamedElementPyobj("func_qualname", needsResolution) ); } @@ -1028,7 +1064,7 @@ class CompilerVisiblePyObj { PyObject_SetAttrString( mPyObject, "__kwdefaults__", - getNamedElementPyobj("func_kwdefaults", needsResolution) + getNamedElementPyobj("func_kwdefaults", needsResolution) ); } @@ -1036,7 +1072,7 @@ class CompilerVisiblePyObj { PyObject_SetAttrString( mPyObject, "__name__", - getNamedElementPyobj("func_name", needsResolution) + getNamedElementPyobj("func_name", needsResolution) ); } } else if (mKind == Kind::PyCodeObject) { @@ -1071,6 +1107,53 @@ class CompilerVisiblePyObj { needsResolution ) ); + } else if (mKind == Kind::PyStaticMethod) { + mPyObject = PyStaticMethod_New(Py_None); + + JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)mPyObject; + decref(method->cm_callable); + method->cm_callable = incref(getNamedElementPyobj("meth_func", needsResolution)); + } else if (mKind == Kind::PyClassMethod) { + static PyObject* nones = PyTuple_Pack(3, Py_None, Py_None, Py_None); + + mPyObject = PyObject_CallObject((PyObject*)&PyProperty_Type, nones); + + JustLikeAPropertyObject* dest = (JustLikeAPropertyObject*)mPyObject; + + decref(dest->prop_get); + decref(dest->prop_set); + decref(dest->prop_del); + decref(dest->prop_doc); + + dest->prop_get = nullptr; + dest->prop_set = nullptr; + dest->prop_del = nullptr; + dest->prop_doc = nullptr; + + #if PY_MINOR_VERSION >= 10 + decref(dest->prop_name); + dest->prop_name = nullptr; + #endif + + dest->prop_get = incref(getNamedElementPyobj("prop_get", needsResolution, true)); + dest->prop_set = incref(getNamedElementPyobj("prop_set", needsResolution, true)); + dest->prop_del = incref(getNamedElementPyobj("prop_del", needsResolution, true)); + dest->prop_doc = incref(getNamedElementPyobj("prop_doc", needsResolution, true)); + + #if PY_MINOR_VERSION >= 10 + decref(dest->prop_name); + dest->prop_name = incref(getNamedElementPyobj("prop_name", needsResolution, true)); + #endif + } else if (mKind == Kind::PyBoundMethod) { + mPyObject = PyMethod_New(Py_None, Py_None); + PyMethodObject* method = (PyMethodObject*)mPyObject; + decref(method->im_func); + decref(method->im_self); + method->im_func = nullptr; + method->im_self = nullptr; + + method->im_func = incref(getNamedElementPyobj("meth_func", needsResolution)); + method->im_self = incref(getNamedElementPyobj("meth_self", needsResolution)); } else { throw std::runtime_error( "Can't make a python object representation for a CVPO of kind " + kindAsString() @@ -1083,10 +1166,14 @@ class CompilerVisiblePyObj { PyObject* getNamedElementPyobj( std::string name, - std::unordered_set& needsResolution + std::unordered_set& needsResolution, + bool allowEmpty=false ) { auto it = mNamedElements.find(name); if (it == mNamedElements.end()) { + if (allowEmpty) { + return nullptr; + } throw std::runtime_error( "Corrupt CompilerVisiblePyObj." + kindAsString() + ". missing " + name ); diff --git a/typed_python/compiler_visible_py_obj_test.py b/typed_python/compiler_visible_py_obj_test.py index dd52bbb6c..6da77ca67 100644 --- a/typed_python/compiler_visible_py_obj_test.py +++ b/typed_python/compiler_visible_py_obj_test.py @@ -28,7 +28,7 @@ def test_make_cvpo_basic(): assert CompilerVisiblePyObj.create(ListOf(int)((1, 2, 3))).kind == 'Instance' assert CompilerVisiblePyObj.create(ListOf(int)((1, 2, 3))).instance[1] == 2 - + def test_cvpo_function(): y = 10 @@ -68,7 +68,7 @@ def f(x: int, z=20): assert cvpo.func_code.co_cellvars.pyobj == f.__code__.co_cellvars assert cvpo.func_code.co_name.pyobj == f.__code__.co_name assert cvpo.func_code.co_filename.pyobj == f.__code__.co_filename - + def test_cvpo_numpy_internals(): assert CompilerVisiblePyObj.create(numpy.array).kind == 'NamedPyObject' @@ -110,11 +110,11 @@ def __init__(self): @staticmethod def staticMeth(): - return 10 + return 10 @classmethod def clsMeth(cls): - return 10 + return 10 gInst = G() gMeth = gInst.f @@ -144,8 +144,8 @@ def clsMeth(cls): GPO = gInstPO.inst_type assert GPO.kind == 'PyClass' assert GPO.cls_bases.pyobj == (C,) - - # we can't actually tell that this is a class dict because + + # we can't actually tell that this is a class dict because # its not the base class assert GPO.cls_dict.kind == 'PyDict' @@ -184,5 +184,7 @@ def f(self): assert C2.__module__ == C.__module__ assert C2.f is not C.f c2Inst = C2cvpo.cls_dict.byKey['f'].func_closure.elements[0].cell_contents.pyobj - + assert C2().f() is c2Inst + + assert CompilerVisiblePyObj.create(print, False).pyobj is print From a0aa6831257426f64ce7f22c83a694b7afa50e02 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 28 Jul 2023 14:37:31 +0000 Subject: [PATCH 62/83] CVPO is complete. --- typed_python/CompilerVisiblePyObj.hpp | 9 ++++- typed_python/PythonTypeInternals.hpp | 41 ++++++++++++++++++++ typed_python/_types.hpp | 21 +--------- typed_python/compiler_visible_py_obj_test.py | 41 ++++++++++++++------ 4 files changed, 80 insertions(+), 32 deletions(-) create mode 100644 typed_python/PythonTypeInternals.hpp diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/CompilerVisiblePyObj.hpp index c72691739..5ef903f4d 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/CompilerVisiblePyObj.hpp @@ -19,6 +19,7 @@ #include #include "Type.hpp" #include "Instance.hpp" +#include "PythonTypeInternals.hpp" /********************************* @@ -717,7 +718,7 @@ class CompilerVisiblePyObj { if (val->ob_type == &PyProperty_Type) { mKind = Kind::PyProperty; - JustLikeAPropertyObject* prop = (JustLikeAPropertyObject*)mPyObject; + JustLikeAPropertyObject* prop = (JustLikeAPropertyObject*)val; if (prop->prop_get) { mNamedElements["prop_get"] = internalize(prop->prop_get); @@ -1114,6 +1115,12 @@ class CompilerVisiblePyObj { decref(method->cm_callable); method->cm_callable = incref(getNamedElementPyobj("meth_func", needsResolution)); } else if (mKind == Kind::PyClassMethod) { + mPyObject = PyClassMethod_New(Py_None); + + JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)mPyObject; + decref(method->cm_callable); + method->cm_callable = incref(getNamedElementPyobj("meth_func", needsResolution)); + } else if (mKind == Kind::PyProperty) { static PyObject* nones = PyTuple_Pack(3, Py_None, Py_None, Py_None); mPyObject = PyObject_CallObject((PyObject*)&PyProperty_Type, nones); diff --git a/typed_python/PythonTypeInternals.hpp b/typed_python/PythonTypeInternals.hpp new file mode 100644 index 000000000..09a242289 --- /dev/null +++ b/typed_python/PythonTypeInternals.hpp @@ -0,0 +1,41 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include + +// the definition of a python property which we can use to read and write its internals +// despite it being an internal detail. +typedef struct { + PyObject_HEAD + PyObject *prop_get; + PyObject *prop_set; + PyObject *prop_del; + PyObject *prop_doc; + + #if PY_MINOR_VERSION >= 10 + PyObject *prop_name; + #endif + + int getter_doc; +} JustLikeAPropertyObject; + +typedef struct { + PyObject_HEAD + PyObject *cm_callable; + PyObject *cm_dict; +} JustLikeAClassOrStaticmethod; diff --git a/typed_python/_types.hpp b/typed_python/_types.hpp index 209e4f420..6244ac00f 100644 --- a/typed_python/_types.hpp +++ b/typed_python/_types.hpp @@ -17,6 +17,7 @@ #pragma once #include +#include "PythonTypeInternals.hpp" PyObject *MakeTupleOrListOfType(PyObject* nullValue, PyObject* args, bool isTuple); PyObject *MakePointerToType(PyObject* nullValue, PyObject* args); @@ -53,23 +54,3 @@ PyObject *MakeClassType(PyObject* nullValue, PyObject* args); PyObject *MakeSubclassOfType(PyObject* nullValue, PyObject* args); PyObject *MakeAlternativeType(PyObject* nullValue, PyObject* args, PyObject* kwargs); PyObject *MakeForwardType(PyObject* nullValue, PyObject* args, PyObject* kwargs); - -typedef struct { - PyObject_HEAD - PyObject *prop_get; - PyObject *prop_set; - PyObject *prop_del; - PyObject *prop_doc; - - #if PY_MINOR_VERSION >= 10 - PyObject *prop_name; - #endif - - int getter_doc; -} JustLikeAPropertyObject; - -typedef struct { - PyObject_HEAD - PyObject *cm_callable; - PyObject *cm_dict; -} JustLikeAClassOrStaticmethod; diff --git a/typed_python/compiler_visible_py_obj_test.py b/typed_python/compiler_visible_py_obj_test.py index 6da77ca67..dc5b804db 100644 --- a/typed_python/compiler_visible_py_obj_test.py +++ b/typed_python/compiler_visible_py_obj_test.py @@ -16,9 +16,9 @@ def test_make_cvpo_basic(): assert CompilerVisiblePyObj.create([1, 2, 3]).kind == 'PyList' assert CompilerVisiblePyObj.create([1, 2, 3]).elements[0].kind == 'Instance' - assert CompilerVisiblePyObj.create({1:2}).kind == 'PyDict' - assert CompilerVisiblePyObj.create({'1':2}).elements[0].kind == 'Instance' - assert CompilerVisiblePyObj.create({'1':2}).keys[0].kind == 'String' + assert CompilerVisiblePyObj.create({1: 2}).kind == 'PyDict' + assert CompilerVisiblePyObj.create({'1': 2}).elements[0].kind == 'Instance' + assert CompilerVisiblePyObj.create({'1': 2}).keys[0].kind == 'String' assert CompilerVisiblePyObj.create(1).kind == 'Instance' assert CompilerVisiblePyObj.create(1).instance == 1 @@ -127,7 +127,7 @@ def clsMeth(cls): assert cvpo.cls_dict.kind == 'PyClassDict' assert cvpo.cls_dict.byKey['aProp'].kind == 'PyProperty' - assert cvpo.cls_dict.byKey['aProp'].prop_fget.kind == 'PyFunction' + assert cvpo.cls_dict.byKey['aProp'].prop_get.kind == 'PyFunction' assert cvpo.cls_dict.byKey['f'].kind == 'PyFunction' @@ -163,7 +163,8 @@ def test_cvpo_rehydration(): assert CompilerVisiblePyObj.create([1, 2, 'hi'], False).pyobj == [1, 2, 'hi'] assert CompilerVisiblePyObj.create((1, 2, 'hi'), False).pyobj == (1, 2, 'hi') assert CompilerVisiblePyObj.create({1, 2}, False).pyobj == {1, 2} - assert CompilerVisiblePyObj.create({1: 2,'3': 4}, False).pyobj == {1: 2, '3': 4} + assert CompilerVisiblePyObj.create({1: 2, '3': 4}, False).pyobj == {1: 2, '3': 4} + assert CompilerVisiblePyObj.create(print, False).pyobj is print def f(): return 1 @@ -171,20 +172,38 @@ def f(): assert CompilerVisiblePyObj.create(f, False).pyobj() == 1 class C: - def f(self): + @property + def aProp(self): + return 10 + + @staticmethod + def aSM(x): + return x + 1 + + @classmethod + def aCM(x): + return x + 1 + + def getCInst(self): return cInst + def getCInstMeth(self): + return cInstMeth + cInst = C() + cInstMeth = cInst.getCInstMeth C2cvpo = CompilerVisiblePyObj.create(C, False) C2 = C2cvpo.pyobj - assert C2 is not C + assert C2 is not C assert C2.__name__ == C.__name__ assert C2.__module__ == C.__module__ - assert C2.f is not C.f - c2Inst = C2cvpo.cls_dict.byKey['f'].func_closure.elements[0].cell_contents.pyobj + assert C2.getCInst is not C.getCInst - assert C2().f() is c2Inst + c2Inst = C2cvpo.cls_dict.byKey['getCInst'].func_closure.elements[0].cell_contents.pyobj + c2InstMeth = C2cvpo.cls_dict.byKey['getCInstMeth'].func_closure.elements[0].cell_contents.pyobj - assert CompilerVisiblePyObj.create(print, False).pyobj is print + assert C2().getCInst() is c2Inst + assert C2().getCInstMeth() is c2InstMeth + assert c2InstMeth() is c2InstMeth From 32380cffef6feb7edb68c5634b777361544d9b5d Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 28 Jul 2023 21:22:17 +0000 Subject: [PATCH 63/83] Rename 'CompilerVisiblePyObj' to 'PyObjSnapshot' as its more general than we were using. --- todo.txt | 34 +++++++- typed_python/FunctionGlobal.hpp | 24 +++--- typed_python/FunctionOverload.hpp | 2 +- typed_python/FunctionType.hpp | 4 +- typed_python/PyFunctionGlobal.cpp | 2 +- ...ilerVisiblePyObj.hpp => PyObjSnapshot.hpp} | 79 +++++++++--------- ...erVisiblePyObj.cpp => PyPyObjSnapshot.cpp} | 72 ++++++++-------- ...erVisiblePyObj.hpp => PyPyObjSnapshot.hpp} | 12 +-- typed_python/Type.cpp | 2 +- typed_python/_types.cpp | 8 +- typed_python/all.cpp | 2 +- ...py_obj_test.py => py_obj_snapshot_test.py} | 82 +++++++++---------- 12 files changed, 174 insertions(+), 149 deletions(-) rename typed_python/{CompilerVisiblePyObj.hpp => PyObjSnapshot.hpp} (94%) rename typed_python/{PyCompilerVisiblePyObj.cpp => PyPyObjSnapshot.cpp} (69%) rename typed_python/{PyCompilerVisiblePyObj.hpp => PyPyObjSnapshot.hpp} (78%) rename typed_python/{compiler_visible_py_obj_test.py => py_obj_snapshot_test.py} (62%) diff --git a/todo.txt b/todo.txt index 8c6316885..ddcacea76 100644 --- a/todo.txt +++ b/todo.txt @@ -24,8 +24,8 @@ - redocument everything - finish serialization - should be able to serialize forward defined types - - ensure we are not leaking CompilerVisiblePyObj values - - really CompilerVisiblePyObj is the wrong name - i think it's more like 'TypeVisiblePyObj' + - ensure we are not leaking PyObjSnapshot values + - really PyObjSnapshot is the wrong name - i think it's more like 'TypeVisiblePyObj' which reflects the reality that some objects are visible to the typing system and some are not - can we get rid of the whole serialize-mrtg thing? at this point, memos should just work right? @@ -47,11 +47,11 @@ - ensure that we don't try to MRTG something that's a forward defined type - fix the compiler to deal with the new globals thing now that we have it well defined - figure out how to properly internalize python classes - - the CompilerVisiblePyObj thing needs to actually work + - the PyObjSnapshot thing needs to actually work - we need to make sure we can actually destroy these if we're not going to keep them - get rid of PyObject throughout Function - should be like a FunctionGlobal -- Value should be holding a CompilerVisiblePyObj, not an instance +- Value should be holding a PyObjSnapshot, not an instance - We're using the TP compiler hash in the compiler, which makes sense. Need to ensure that the CVOV obyes the same rules for dotted accesses that the original compiler does. - We need to ensure that Globals retain their source information if its available @@ -62,6 +62,32 @@ for things like lists/classes/etc +strategy: + * fully fleshed out CVPO - infrastructure to help us manage all of this: + * CVPO is really 'PyObjSnapshot' and represents a frozen graph of python objects + * CVPO has an 'owner' which manages its lifetime. + * there's an 'internalized' owner which represents CVPO that's owned by the core and that has been internalized by hash + * these are leaked + * CVPO models instances and Types all the way out + * that is, a 'Type' is actually modeled with its pieces, which also have CVPO + * the 'Type' and 'mPyObject' are pointers to the real thing + * when you cross into a CVPO on another 'owner' then you can stop + * instances are the same way, including classes, etc. + * we replicate the MRTG hashing functionality on CVPO so that you can hash the items in a CVPO + * you're not forced to internalize if you do this + * when we freeze a Forward, we walk it to make a CVPO graph and hash it. + * we then internalize the new part of the graph + * then we walk back over the original objects and mark them as resolving to the new ones + * then we autoresolve to update forwards that point to Cells and Modules + * serialization + * when we serialize a forward type, we just do it normally + * but when we serialize a resolved type we serialize its CVPO graph + * then when we deserialize it, we can deserialize the graph and then internalize it if we want to + * this lets us gracefully handle the 'leak' case + + + + question: how should we handle regular python classes defined in forwards? * they will trigger MutuallyRecursiveTypeGroup and identityHash, which should be fine. * we've said that Forwards shouldn't participate in MRTG. diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 4e7a79880..154787059 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -17,7 +17,7 @@ #pragma once -#include "CompilerVisiblePyObj.hpp" +#include "PyObjSnapshot.hpp" class FunctionGlobal { @@ -34,7 +34,7 @@ class FunctionGlobal { PyObject* dictOrCell, std::string name, std::string moduleName, - CompilerVisiblePyObj* constant + PyObjSnapshot* constant ) : mKind(inKind), mModuleDictOrCell(incref(dictOrCell)), @@ -79,7 +79,7 @@ class FunctionGlobal { return FunctionGlobal(); } - static FunctionGlobal Constant(CompilerVisiblePyObj* constant) { + static FunctionGlobal Constant(PyObjSnapshot* constant) { return FunctionGlobal( GlobalType::Constant, nullptr, @@ -202,7 +202,7 @@ class FunctionGlobal { return mKind; } - CompilerVisiblePyObj* getConstant() const { + PyObjSnapshot* getConstant() const { return mConstant; } @@ -323,7 +323,7 @@ class FunctionGlobal { } FunctionGlobal withConstantsInternalized( - std::unordered_map& constantMapCache, + std::unordered_map& constantMapCache, const std::map& groupMap ) { if (!(isGlobalInDict() || isGlobalInCell())) { @@ -335,11 +335,11 @@ class FunctionGlobal { auto it = groupMap.find(t); if (it != groupMap.end()) { return FunctionGlobal::Constant( - CompilerVisiblePyObj::ForType(it->second) + PyObjSnapshot::ForType(it->second) ); } else { return FunctionGlobal::Constant( - CompilerVisiblePyObj::ForType(t) + PyObjSnapshot::ForType(t) ); } } @@ -351,7 +351,7 @@ class FunctionGlobal { } return FunctionGlobal::Constant( - CompilerVisiblePyObj::internalizePyObj( + PyObjSnapshot::internalizePyObj( value, constantMapCache, groupMap @@ -371,7 +371,7 @@ class FunctionGlobal { if (it != groupMap.end()) { // we're actually a constant! return FunctionGlobal::Constant( - CompilerVisiblePyObj::ForType(it->second) + PyObjSnapshot::ForType(it->second) ); } @@ -477,7 +477,7 @@ class FunctionGlobal { uint64_t kind = 0; std::string name, moduleName; PyObjectHolder cellOrModule; - CompilerVisiblePyObj* constant = nullptr; + PyObjSnapshot* constant = nullptr; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { @@ -493,7 +493,7 @@ class FunctionGlobal { moduleName = buffer.readStringObject(); } else if (fieldNumber == 4) { - constant = CompilerVisiblePyObj::deserialize(context, buffer, wireType); + constant = PyObjSnapshot::deserialize(context, buffer, wireType); } }); @@ -563,7 +563,7 @@ class FunctionGlobal { private: GlobalType mKind; - CompilerVisiblePyObj* mConstant; + PyObjSnapshot* mConstant; std::string mName; std::string mModuleName; diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 3ed580df1..466c77ba0 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -278,7 +278,7 @@ class FunctionOverload { } void internalizeConstants( - std::unordered_map& constantMapCache, + std::unordered_map& constantMapCache, const std::map& groupMap ) { for (auto& nameAndGlobal: mGlobals) { diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index 59a869ae6..c91e9196f 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -159,9 +159,9 @@ class Function : public Type { } // replace any direct references to PyObject we're holding internally - // with CompilerVisiblePyObj references instead + // with PyObjSnapshot references instead void internalizeConstants( - std::unordered_map& constantMapCache, + std::unordered_map& constantMapCache, const std::map& groupMap ) { for (auto& o: mOverloads) { diff --git a/typed_python/PyFunctionGlobal.cpp b/typed_python/PyFunctionGlobal.cpp index c02c13da1..37a0afd2e 100644 --- a/typed_python/PyFunctionGlobal.cpp +++ b/typed_python/PyFunctionGlobal.cpp @@ -160,7 +160,7 @@ PyObject* PyFunctionGlobal::tp_getattro(PyObject* selfObj, PyObject* attrName) { } if (attr == "constant" && global.isConstant()) { - return PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(global.getConstant()); + return PyPyObjSnapshot::newPyObjSnapshot(global.getConstant()); } if (attr == "name" && (global.isNamedModuleMember() || global.isGlobalInDict())) { diff --git a/typed_python/CompilerVisiblePyObj.hpp b/typed_python/PyObjSnapshot.hpp similarity index 94% rename from typed_python/CompilerVisiblePyObj.hpp rename to typed_python/PyObjSnapshot.hpp index 5ef903f4d..af0eb36e0 100644 --- a/typed_python/CompilerVisiblePyObj.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -23,14 +23,13 @@ /********************************* -CompilerVisiblePyObject +PyObjSnapshot -a representation of a python object that's owned by TypedPython. We hold these by -pointer and leak them indiscriminately - like Type objects they're considered to be permanent -and singletonish. - -They are intended to be a 'snapshot' of the state of a collection of python objects and -contain enough information for the compiler to use them to build compiled code. +A representation of a python object that's owned by TypedPython. They are intended to be a +'snapshot' of the state of a collection of python objects and contain enough information +for the compiler to use them to build compiled code, and for us to rebuild the object +entirely from the snapshot. This gives us a scaffolding to describe objects and object +graphs independent of the state of the python interpreter. To the extent that they visit 'mutable' objects like a list (which might be contained within a default argument), they will encode the state of the object when it was first seen. This @@ -39,7 +38,7 @@ gives us a self-consistent view of the world to compile against so we don't have changing underneath us. **********************************/ -class CompilerVisiblePyObj { +class PyObjSnapshot { enum class Kind { // this should never be visible in a running program Uninitialized = 0, @@ -93,7 +92,7 @@ class CompilerVisiblePyObj { ArbitraryPyObject }; - CompilerVisiblePyObj() : + PyObjSnapshot() : mKind(Kind::Uninitialized), mType(nullptr), mPyObject(nullptr) @@ -101,8 +100,8 @@ class CompilerVisiblePyObj { } public: - static CompilerVisiblePyObj* ForType(Type* t) { - CompilerVisiblePyObj* res = new CompilerVisiblePyObj(); + static PyObjSnapshot* ForType(Type* t) { + PyObjSnapshot* res = new PyObjSnapshot(); res->mKind = Kind::Type; res->mType = t; return res; @@ -261,7 +260,7 @@ class CompilerVisiblePyObj { return "ArbitraryPyObject"; } - throw std::runtime_error("Unknown CompilerVisiblePyObj Kind"); + throw std::runtime_error("Unknown PyObjSnapshot Kind"); } static std::string dictGetStringOrEmpty(PyObject* dict, const char* name) { @@ -287,9 +286,9 @@ class CompilerVisiblePyObj { // return a CVPO for 'val', stashing it in 'constantMapCache' // in case we hit a recursion. - static CompilerVisiblePyObj* internalizePyObj( + static PyObjSnapshot* internalizePyObj( PyObject* val, - std::unordered_map& constantMapCache, + std::unordered_map& constantMapCache, const std::map<::Type*, ::Type*>& groupMap, bool linkBackToOriginalObject = true ) { @@ -299,7 +298,7 @@ class CompilerVisiblePyObj { return it->second; } - constantMapCache[val] = new CompilerVisiblePyObj(); + constantMapCache[val] = new PyObjSnapshot(); constantMapCache[val]->becomeInternalizedOf( val, constantMapCache, groupMap, linkBackToOriginalObject @@ -369,7 +368,7 @@ class CompilerVisiblePyObj { void becomeInternalizedOf( PyObject* val, - std::unordered_map& constantMapCache, + std::unordered_map& constantMapCache, const std::map<::Type*, ::Type*>& groupMap, bool linkBackToOriginalObject ) { @@ -379,7 +378,7 @@ class CompilerVisiblePyObj { } auto internalize = [&](PyObject* o) { - return CompilerVisiblePyObj::internalizePyObj( + return PyObjSnapshot::internalizePyObj( o, constantMapCache, groupMap, linkBackToOriginalObject ); }; @@ -771,7 +770,7 @@ class CompilerVisiblePyObj { mPyObject = incref(val); } - void append(CompilerVisiblePyObj* elt) { + void append(PyObjSnapshot* elt) { if (mKind != Kind::PyTuple) { throw std::runtime_error("Expected a PyTuple"); } @@ -779,15 +778,15 @@ class CompilerVisiblePyObj { mElements.push_back(elt); } - const std::vector& elements() const { + const std::vector& elements() const { return mElements; } - const std::vector& keys() const { + const std::vector& keys() const { return mKeys; } - const std::map& namedElements() const { + const std::map& namedElements() const { return mNamedElements; } @@ -844,7 +843,7 @@ class CompilerVisiblePyObj { if (mKind == Kind::PyTuple) { // TODO: what to do here? - throw std::runtime_error("TODO: CompilerVisiblePyObj::_visitCompilerVisibleInternals PyTuple"); + throw std::runtime_error("TODO: PyObjSnapshot::_visitCompilerVisibleInternals PyTuple"); } if (mKind == Kind::ArbitraryPyObject) { @@ -861,7 +860,7 @@ class CompilerVisiblePyObj { return mPyObject; } - std::unordered_set needsResolution; + std::unordered_set needsResolution; PyObject* res = getPyObj(needsResolution); @@ -884,7 +883,7 @@ class CompilerVisiblePyObj { throw std::runtime_error("Corrupt PyClass - no cls_dict"); } - CompilerVisiblePyObj* clsDictPyObj = mNamedElements["cls_dict"]; + PyObjSnapshot* clsDictPyObj = mNamedElements["cls_dict"]; for (long k = 0; k < clsDictPyObj->elements().size() && k < clsDictPyObj->keys().size(); k++) { if (clsDictPyObj->keys()[k]->isString()) { @@ -919,12 +918,12 @@ class CompilerVisiblePyObj { } } - PyObject* getPyObj(std::unordered_set& needsResolution) { + PyObject* getPyObj(std::unordered_set& needsResolution) { PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); if (!mPyObject) { if (mKind == Kind::ArbitraryPyObject) { - throw std::runtime_error("Corrupt CompilerVisiblePyObj.ArbitraryPyObject: missing mPyObject"); + throw std::runtime_error("Corrupt PyObjSnapshot.ArbitraryPyObject: missing mPyObject"); } else if (mKind == Kind::Type) { mPyObject = (PyObject*)PyInstance::typeObj(mType); } else if (mKind == Kind::String) { @@ -1173,7 +1172,7 @@ class CompilerVisiblePyObj { PyObject* getNamedElementPyobj( std::string name, - std::unordered_set& needsResolution, + std::unordered_set& needsResolution, bool allowEmpty=false ) { auto it = mNamedElements.find(name); @@ -1182,7 +1181,7 @@ class CompilerVisiblePyObj { return nullptr; } throw std::runtime_error( - "Corrupt CompilerVisiblePyObj." + kindAsString() + ". missing " + name + "Corrupt PyObjSnapshot." + kindAsString() + ". missing " + name ); } @@ -1190,7 +1189,7 @@ class CompilerVisiblePyObj { } std::string toString() const { - return "CompilerVisiblePyObj." + kindAsString() + "()"; + return "PyObjSnapshot." + kindAsString() + "()"; } template @@ -1234,16 +1233,16 @@ class CompilerVisiblePyObj { } template - static CompilerVisiblePyObj* deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { + static PyObjSnapshot* deserialize(serialization_context_t& context, buf_t& buffer, int wireType) { int64_t kind = 0; ::Type* type = nullptr; - std::vector vec; + std::vector vec; ::Instance i; uint32_t id = -1; PyObjectHolder pyobj; - CompilerVisiblePyObj* result = nullptr; + PyObjSnapshot* result = nullptr; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { @@ -1251,9 +1250,9 @@ class CompilerVisiblePyObj { void* ptr = buffer.lookupCachedPointer(id); if (ptr) { - result = (CompilerVisiblePyObj*)result; + result = (PyObjSnapshot*)result; } else { - result = new CompilerVisiblePyObj(); + result = new PyObjSnapshot(); buffer.addCachedPointer(id, result, nullptr); } } else @@ -1268,7 +1267,7 @@ class CompilerVisiblePyObj { } else if (fieldNumber == 4) { buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { - vec.push_back(CompilerVisiblePyObj::deserialize(context, buffer, wireType)); + vec.push_back(PyObjSnapshot::deserialize(context, buffer, wireType)); }); } else if (fieldNumber == 5) { @@ -1277,11 +1276,11 @@ class CompilerVisiblePyObj { }); if (!result) { - throw std::runtime_error("corrupt CompilerVisiblePyObj - no memo found"); + throw std::runtime_error("corrupt PyObjSnapshot - no memo found"); } if (kind == -1) { - throw std::runtime_error("corrupt CompilerVisiblePyObj - invalid kind"); + throw std::runtime_error("corrupt PyObjSnapshot - invalid kind"); } result->mKind = Kind(kind); @@ -1319,14 +1318,14 @@ class CompilerVisiblePyObj { // otherwise, it will be a cache for a constructed instance of the object PyObject* mPyObject; - std::map mNamedElements; + std::map mNamedElements; std::map mNamedInts; // if we're a tuple, list, or dict - std::vector mElements; + std::vector mElements; // if we're a PyDict - std::vector mKeys; + std::vector mKeys; std::string mStringValue; std::string mModuleName; diff --git a/typed_python/PyCompilerVisiblePyObj.cpp b/typed_python/PyPyObjSnapshot.cpp similarity index 69% rename from typed_python/PyCompilerVisiblePyObj.cpp rename to typed_python/PyPyObjSnapshot.cpp index 9cc76572e..82418caf7 100644 --- a/typed_python/PyCompilerVisiblePyObj.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -14,21 +14,21 @@ limitations under the License. ******************************************************************************/ -#include "PyCompilerVisiblePyObj.hpp" +#include "PyPyObjSnapshot.hpp" -PyDoc_STRVAR(PyCompilerVisiblePyObj_doc, +PyDoc_STRVAR(PyPyObjSnapshot_doc, "A single overload of a Function type object.\n\n" ); -PyMethodDef PyCompilerVisiblePyObj_methods[] = { - {"create", (PyCFunction)PyCompilerVisiblePyObj::create, METH_VARARGS | METH_KEYWORDS | METH_STATIC, NULL}, +PyMethodDef PyPyObjSnapshot_methods[] = { + {"create", (PyCFunction)PyPyObjSnapshot::create, METH_VARARGS | METH_KEYWORDS | METH_STATIC, NULL}, {NULL} /* Sentinel */ }; /* static */ -PyObject* PyCompilerVisiblePyObj::create(PyObject* self, PyObject* args, PyObject* kwargs) { +PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwargs) { return translateExceptionToPyObject([&]() { PyObject* instance; int linkBack = 1; @@ -38,37 +38,37 @@ PyObject* PyCompilerVisiblePyObj::create(PyObject* self, PyObject* args, PyObjec throw PythonExceptionSet(); } - std::unordered_map constantMapCache; + std::unordered_map constantMapCache; std::map<::Type*, ::Type*> groupMap; - CompilerVisiblePyObj* object = CompilerVisiblePyObj::internalizePyObj( + PyObjSnapshot* object = PyObjSnapshot::internalizePyObj( instance, constantMapCache, groupMap, linkBack ); - return PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(object); + return PyPyObjSnapshot::newPyObjSnapshot(object); }); } /* static */ -void PyCompilerVisiblePyObj::dealloc(PyCompilerVisiblePyObj *self) +void PyPyObjSnapshot::dealloc(PyPyObjSnapshot *self) { decref(self->mElements); decref(self->mKeys); Py_TYPE(self)->tp_free((PyObject*)self); } -PyObject* PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(CompilerVisiblePyObj* g) { - static std::unordered_map memo; +PyObject* PyPyObjSnapshot::newPyObjSnapshot(PyObjSnapshot* g) { + static std::unordered_map memo; auto it = memo.find(g); if (it != memo.end()) { return incref(it->second); } - PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)PyType_CompilerVisiblePyObj.tp_alloc(&PyType_CompilerVisiblePyObj, 0); + PyPyObjSnapshot* self = (PyPyObjSnapshot*)PyType_PyObjSnapshot.tp_alloc(&PyType_PyObjSnapshot, 0); self->mPyobj = g; self->mElements = nullptr; self->mKeys = nullptr; @@ -80,11 +80,11 @@ PyObject* PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(CompilerVisiblePyObj } /* static */ -PyObject* PyCompilerVisiblePyObj::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +PyObject* PyPyObjSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - PyCompilerVisiblePyObj* self; + PyPyObjSnapshot* self; - self = (PyCompilerVisiblePyObj*)type->tp_alloc(type, 0); + self = (PyPyObjSnapshot*)type->tp_alloc(type, 0); if (self != NULL) { self->mPyobj = nullptr; @@ -96,8 +96,8 @@ PyObject* PyCompilerVisiblePyObj::new_(PyTypeObject *type, PyObject *args, PyObj return (PyObject*)self; } -PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrName) { - PyCompilerVisiblePyObj* pyCVPO = (PyCompilerVisiblePyObj*)selfObj; +PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { + PyPyObjSnapshot* pyCVPO = (PyPyObjSnapshot*)selfObj; return translateExceptionToPyObject([&] { if (!PyUnicode_Check(attrName)) { @@ -106,11 +106,11 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN std::string attr(PyUnicode_AsUTF8(attrName)); - CompilerVisiblePyObj* obj = pyCVPO->mPyobj; + PyObjSnapshot* obj = pyCVPO->mPyobj; auto it = obj->namedElements().find(attr); if (it != obj->namedElements().end()) { - return PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(it->second); + return PyPyObjSnapshot::newPyObjSnapshot(it->second); } auto it2 = obj->namedInts().find(attr); @@ -154,7 +154,7 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN if (!pyCVPO->mElements) { pyCVPO->mElements = PyList_New(0); for (auto p: obj->elements()) { - PyList_Append(pyCVPO->mElements, PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(p)); + PyList_Append(pyCVPO->mElements, PyPyObjSnapshot::newPyObjSnapshot(p)); } } @@ -165,7 +165,7 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN if (!pyCVPO->mKeys) { pyCVPO->mKeys = PyList_New(0); for (auto p: obj->keys()) { - PyList_Append(pyCVPO->mKeys, PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(p)); + PyList_Append(pyCVPO->mKeys, PyPyObjSnapshot::newPyObjSnapshot(p)); } } @@ -179,7 +179,7 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN PyDict_SetItem( pyCVPO->mByKey, obj->keys()[k]->getPyObj(), - PyCompilerVisiblePyObj::newPyCompilerVisiblePyObj(obj->elements()[k]) + PyPyObjSnapshot::newPyObjSnapshot(obj->elements()[k]) ); } } @@ -191,8 +191,8 @@ PyObject* PyCompilerVisiblePyObj::tp_getattro(PyObject* selfObj, PyObject* attrN }); } -PyObject* PyCompilerVisiblePyObj::tp_repr(PyObject *selfObj) { - PyCompilerVisiblePyObj* self = (PyCompilerVisiblePyObj*)selfObj; +PyObject* PyPyObjSnapshot::tp_repr(PyObject *selfObj) { + PyPyObjSnapshot* self = (PyPyObjSnapshot*)selfObj; return translateExceptionToPyObject([&]() { return PyUnicode_FromString(self->mPyobj->toString().c_str()); @@ -201,19 +201,19 @@ PyObject* PyCompilerVisiblePyObj::tp_repr(PyObject *selfObj) { /* static */ -int PyCompilerVisiblePyObj::init(PyCompilerVisiblePyObj *self, PyObject *args, PyObject *kwargs) +int PyPyObjSnapshot::init(PyPyObjSnapshot *self, PyObject *args, PyObject *kwargs) { - PyErr_Format(PyExc_RuntimeError, "CompilerVisiblePyObj cannot be initialized directly"); + PyErr_Format(PyExc_RuntimeError, "PyObjSnapshot cannot be initialized directly"); return -1; } -PyTypeObject PyType_CompilerVisiblePyObj = { +PyTypeObject PyType_PyObjSnapshot = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "CompilerVisiblePyObj", - .tp_basicsize = sizeof(PyCompilerVisiblePyObj), + .tp_name = "PyObjSnapshot", + .tp_basicsize = sizeof(PyPyObjSnapshot), .tp_itemsize = 0, - .tp_dealloc = (destructor) PyCompilerVisiblePyObj::dealloc, + .tp_dealloc = (destructor) PyPyObjSnapshot::dealloc, #if PY_MINOR_VERSION < 8 .tp_print = 0, #else @@ -222,25 +222,25 @@ PyTypeObject PyType_CompilerVisiblePyObj = { .tp_getattr = 0, .tp_setattr = 0, .tp_as_async = 0, - .tp_repr = PyCompilerVisiblePyObj::tp_repr, + .tp_repr = PyPyObjSnapshot::tp_repr, .tp_as_number = 0, .tp_as_sequence = 0, .tp_as_mapping = 0, .tp_hash = 0, .tp_call = 0, .tp_str = 0, - .tp_getattro = PyCompilerVisiblePyObj::tp_getattro, + .tp_getattro = PyPyObjSnapshot::tp_getattro, .tp_setattro = 0, .tp_as_buffer = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = PyCompilerVisiblePyObj_doc, + .tp_doc = PyPyObjSnapshot_doc, .tp_traverse = 0, .tp_clear = 0, .tp_richcompare = 0, .tp_weaklistoffset = 0, .tp_iter = 0, .tp_iternext = 0, - .tp_methods = PyCompilerVisiblePyObj_methods, + .tp_methods = PyPyObjSnapshot_methods, .tp_members = 0, .tp_getset = 0, .tp_base = 0, @@ -248,9 +248,9 @@ PyTypeObject PyType_CompilerVisiblePyObj = { .tp_descr_get = 0, .tp_descr_set = 0, .tp_dictoffset = 0, - .tp_init = (initproc) PyCompilerVisiblePyObj::init, + .tp_init = (initproc) PyPyObjSnapshot::init, .tp_alloc = 0, - .tp_new = PyCompilerVisiblePyObj::new_, + .tp_new = PyPyObjSnapshot::new_, .tp_free = 0, .tp_is_gc = 0, .tp_bases = 0, diff --git a/typed_python/PyCompilerVisiblePyObj.hpp b/typed_python/PyPyObjSnapshot.hpp similarity index 78% rename from typed_python/PyCompilerVisiblePyObj.hpp rename to typed_python/PyPyObjSnapshot.hpp index 653cbfc75..d643f6c08 100644 --- a/typed_python/PyCompilerVisiblePyObj.hpp +++ b/typed_python/PyPyObjSnapshot.hpp @@ -18,11 +18,11 @@ #include "PyInstance.hpp" -class PyCompilerVisiblePyObj { +class PyPyObjSnapshot { public: PyObject_HEAD - CompilerVisiblePyObj* mPyobj; + PyObjSnapshot* mPyobj; PyObject* mElements; PyObject* mKeys; PyObject* mByKey; @@ -33,13 +33,13 @@ class PyCompilerVisiblePyObj { static PyObject* tp_getattro(PyObject* selfObj, PyObject* attr); - static void dealloc(PyCompilerVisiblePyObj *self); + static void dealloc(PyPyObjSnapshot *self); static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); - static PyObject* newPyCompilerVisiblePyObj(CompilerVisiblePyObj* o); + static PyObject* newPyObjSnapshot(PyObjSnapshot* o); - static int init(PyCompilerVisiblePyObj *self, PyObject *args, PyObject *kwargs); + static int init(PyPyObjSnapshot *self, PyObject *args, PyObject *kwargs); }; -extern PyTypeObject PyType_CompilerVisiblePyObj; +extern PyTypeObject PyType_PyObjSnapshot; diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 3fd8902ce..2a9d4fd9f 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -476,7 +476,7 @@ void Type::attemptToResolve() { // allow each Function type to build CompilerVisiblePythonObjects for its globals // after this, any FunctionGlobals will refer to Constants not cells - std::unordered_map compilerVisiblePyObjMap; + std::unordered_map compilerVisiblePyObjMap; for (auto typeAndSource: resolutionSource) { if (typeAndSource.first->isFunction()) { ((Function*)typeAndSource.first)->internalizeConstants( diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 59aaf0295..a059bd98b 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -37,7 +37,7 @@ #include "PySlab.hpp" #include "PyFunctionOverload.hpp" #include "PyFunctionGlobal.hpp" -#include "PyCompilerVisiblePyObj.hpp" +#include "PyPyObjSnapshot.hpp" #include "PyModuleRepresentation.hpp" #include "_types.hpp" #include "CompilerVisibleObjectVisitor.hpp" @@ -2772,7 +2772,7 @@ PyObject *resolveForwardDefinedType(PyObject* nullValue, PyObject* args) { if (PyTuple_Size(args) != 1) { throw std::runtime_error("resolveForwardDefinedType takes 1 positional argument"); } - + PyObjectHolder a1(PyTuple_GetItem(args, 0)); Type* t = PyInstance::unwrapTypeArgToTypePtr(a1); @@ -3859,12 +3859,12 @@ PyInit__types(void) return NULL; } - if (PyType_Ready(&PyType_CompilerVisiblePyObj) < 0) { + if (PyType_Ready(&PyType_PyObjSnapshot) < 0) { return NULL; } PyModule_AddObject(module, "FunctionOverload", (PyObject*)incref(&PyType_FunctionOverload)); - PyModule_AddObject(module, "CompilerVisiblePyObj", (PyObject*)incref(&PyType_CompilerVisiblePyObj)); + PyModule_AddObject(module, "PyObjSnapshot", (PyObject*)incref(&PyType_PyObjSnapshot)); PyModule_AddObject(module, "FunctionGlobal", (PyObject*)incref(&PyType_FunctionGlobal)); PyModule_AddObject(module, "Slab", (PyObject*)incref(&PyType_Slab)); PyModule_AddObject(module, "ModuleRepresentation", (PyObject*)incref(&PyType_ModuleRepresentation)); diff --git a/typed_python/all.cpp b/typed_python/all.cpp index a0a281b27..842bdcb47 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -78,7 +78,7 @@ compile the entire group all at once. #include "PyTemporaryReferenceTracer.cpp" #include "PyFunctionOverload.cpp" #include "PyFunctionGlobal.cpp" -#include "PyCompilerVisiblePyObj.cpp" +#include "PyPyObjSnapshot.cpp" #include "lz4.c" #include "lz4frame.c" diff --git a/typed_python/compiler_visible_py_obj_test.py b/typed_python/py_obj_snapshot_test.py similarity index 62% rename from typed_python/compiler_visible_py_obj_test.py rename to typed_python/py_obj_snapshot_test.py index dc5b804db..0eac66a6d 100644 --- a/typed_python/compiler_visible_py_obj_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -1,33 +1,33 @@ import numpy from typed_python import ListOf -from typed_python._types import CompilerVisiblePyObj +from typed_python._types import PyObjSnapshot def test_make_cvpo_basic(): - assert CompilerVisiblePyObj.create(int).kind == 'Type' + assert PyObjSnapshot.create(int).kind == 'Type' - assert CompilerVisiblePyObj.create((1, 2, 3)).kind == 'PyTuple' - assert CompilerVisiblePyObj.create((1, 2, 3)).elements[0].kind == 'Instance' + assert PyObjSnapshot.create((1, 2, 3)).kind == 'PyTuple' + assert PyObjSnapshot.create((1, 2, 3)).elements[0].kind == 'Instance' - assert CompilerVisiblePyObj.create({1, 2, 3}).kind == 'PySet' - assert CompilerVisiblePyObj.create({1, 2, 3}).elements[0].kind == 'Instance' + assert PyObjSnapshot.create({1, 2, 3}).kind == 'PySet' + assert PyObjSnapshot.create({1, 2, 3}).elements[0].kind == 'Instance' - assert CompilerVisiblePyObj.create([1, 2, 3]).kind == 'PyList' - assert CompilerVisiblePyObj.create([1, 2, 3]).elements[0].kind == 'Instance' + assert PyObjSnapshot.create([1, 2, 3]).kind == 'PyList' + assert PyObjSnapshot.create([1, 2, 3]).elements[0].kind == 'Instance' - assert CompilerVisiblePyObj.create({1: 2}).kind == 'PyDict' - assert CompilerVisiblePyObj.create({'1': 2}).elements[0].kind == 'Instance' - assert CompilerVisiblePyObj.create({'1': 2}).keys[0].kind == 'String' + assert PyObjSnapshot.create({1: 2}).kind == 'PyDict' + assert PyObjSnapshot.create({'1': 2}).elements[0].kind == 'Instance' + assert PyObjSnapshot.create({'1': 2}).keys[0].kind == 'String' - assert CompilerVisiblePyObj.create(1).kind == 'Instance' - assert CompilerVisiblePyObj.create(1).instance == 1 + assert PyObjSnapshot.create(1).kind == 'Instance' + assert PyObjSnapshot.create(1).instance == 1 - assert CompilerVisiblePyObj.create('1').kind == 'String' - assert CompilerVisiblePyObj.create('1').stringValue == '1' + assert PyObjSnapshot.create('1').kind == 'String' + assert PyObjSnapshot.create('1').stringValue == '1' - assert CompilerVisiblePyObj.create(ListOf(int)((1, 2, 3))).kind == 'Instance' - assert CompilerVisiblePyObj.create(ListOf(int)((1, 2, 3))).instance[1] == 2 + assert PyObjSnapshot.create(ListOf(int)((1, 2, 3))).kind == 'Instance' + assert PyObjSnapshot.create(ListOf(int)((1, 2, 3))).instance[1] == 2 def test_cvpo_function(): @@ -36,18 +36,18 @@ def test_cvpo_function(): def f(x: int, z=20): return 10 + y - cvpo = CompilerVisiblePyObj.create(f) + cvpo = PyObjSnapshot.create(f) assert cvpo.kind == 'PyFunction' assert cvpo.name == 'f' - assert cvpo.moduleName == 'typed_python.compiler_visible_py_obj_test' + assert cvpo.moduleName == 'typed_python.py_obj_snapshot_test' assert cvpo.func_name.kind == 'String' assert cvpo.func_module.kind == 'String' assert cvpo.func_closure.kind == 'PyTuple' assert cvpo.func_closure.elements[0].kind == 'PyCell' assert cvpo.func_closure.elements[0].cell_contents.pyobj == 10 assert cvpo.func_globals.kind == "PyModuleDict" - assert cvpo.func_globals.name == 'typed_python.compiler_visible_py_obj_test' + assert cvpo.func_globals.name == 'typed_python.py_obj_snapshot_test' assert cvpo.func_annotations.pyobj['x'] is int assert cvpo.func_defaults.pyobj == (20,) assert not hasattr(cvpo, 'func_kwdefaults') @@ -71,7 +71,7 @@ def f(x: int, z=20): def test_cvpo_numpy_internals(): - assert CompilerVisiblePyObj.create(numpy.array).kind == 'NamedPyObject' + assert PyObjSnapshot.create(numpy.array).kind == 'NamedPyObject' # in theory, we could do better than this by using the reduce pathway. # however, that would involve calling into arbitrary python code during walk time @@ -79,17 +79,17 @@ def test_cvpo_numpy_internals(): # (b) it can release the GIL allowing other threads to run (who might then # modify objects) all of which make it impossible to get a good trace of the # graph. - assert CompilerVisiblePyObj.create(numpy.sin).kind == 'ArbitraryPyObject' - assert CompilerVisiblePyObj.create(numpy.array([1])).kind == 'ArbitraryPyObject' + assert PyObjSnapshot.create(numpy.sin).kind == 'ArbitraryPyObject' + assert PyObjSnapshot.create(numpy.array([1])).kind == 'ArbitraryPyObject' def test_cvpo_builtins(): - assert CompilerVisiblePyObj.create(__builtins__).kind == 'PyModuleDict' - assert CompilerVisiblePyObj.create(__builtins__).module_dict_of.kind == 'PyModule' - assert CompilerVisiblePyObj.create(set).kind == 'NamedPyObject' - assert CompilerVisiblePyObj.create(print).kind == 'NamedPyObject' - assert CompilerVisiblePyObj.create(len).kind == 'NamedPyObject' - assert CompilerVisiblePyObj.create(range).kind == 'NamedPyObject' + assert PyObjSnapshot.create(__builtins__).kind == 'PyModuleDict' + assert PyObjSnapshot.create(__builtins__).module_dict_of.kind == 'PyModule' + assert PyObjSnapshot.create(set).kind == 'NamedPyObject' + assert PyObjSnapshot.create(print).kind == 'NamedPyObject' + assert PyObjSnapshot.create(len).kind == 'NamedPyObject' + assert PyObjSnapshot.create(range).kind == 'NamedPyObject' def test_cvpo_class(): @@ -119,11 +119,11 @@ def clsMeth(cls): gInst = G() gMeth = gInst.f - cvpo = CompilerVisiblePyObj.create(C) + cvpo = PyObjSnapshot.create(C) assert cvpo.kind == 'PyClass' assert cvpo.name == 'C' - assert cvpo.moduleName == 'typed_python.compiler_visible_py_obj_test' + assert cvpo.moduleName == 'typed_python.py_obj_snapshot_test' assert cvpo.cls_dict.kind == 'PyClassDict' assert cvpo.cls_dict.byKey['aProp'].kind == 'PyProperty' @@ -157,19 +157,19 @@ def clsMeth(cls): def test_cvpo_rehydration(): - assert CompilerVisiblePyObj.create(1, False).pyobj == 1 - assert CompilerVisiblePyObj.create('1', False).pyobj == '1' - assert CompilerVisiblePyObj.create(b'1', False).pyobj == b'1' - assert CompilerVisiblePyObj.create([1, 2, 'hi'], False).pyobj == [1, 2, 'hi'] - assert CompilerVisiblePyObj.create((1, 2, 'hi'), False).pyobj == (1, 2, 'hi') - assert CompilerVisiblePyObj.create({1, 2}, False).pyobj == {1, 2} - assert CompilerVisiblePyObj.create({1: 2, '3': 4}, False).pyobj == {1: 2, '3': 4} - assert CompilerVisiblePyObj.create(print, False).pyobj is print + assert PyObjSnapshot.create(1, False).pyobj == 1 + assert PyObjSnapshot.create('1', False).pyobj == '1' + assert PyObjSnapshot.create(b'1', False).pyobj == b'1' + assert PyObjSnapshot.create([1, 2, 'hi'], False).pyobj == [1, 2, 'hi'] + assert PyObjSnapshot.create((1, 2, 'hi'), False).pyobj == (1, 2, 'hi') + assert PyObjSnapshot.create({1, 2}, False).pyobj == {1, 2} + assert PyObjSnapshot.create({1: 2, '3': 4}, False).pyobj == {1: 2, '3': 4} + assert PyObjSnapshot.create(print, False).pyobj is print def f(): return 1 - assert CompilerVisiblePyObj.create(f, False).pyobj() == 1 + assert PyObjSnapshot.create(f, False).pyobj() == 1 class C: @property @@ -193,7 +193,7 @@ def getCInstMeth(self): cInst = C() cInstMeth = cInst.getCInstMeth - C2cvpo = CompilerVisiblePyObj.create(C, False) + C2cvpo = PyObjSnapshot.create(C, False) C2 = C2cvpo.pyobj assert C2 is not C From f3fd5abf07ea324183da327f5291a543a82b45ed Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 28 Jul 2023 22:17:13 +0000 Subject: [PATCH 64/83] Start to organize snapshots into graphs, so we can track them. --- typed_python/PyObjGraphSnapshot.hpp | 65 ++++++++++++ typed_python/PyObjSnapshot.cpp | 30 ++++++ typed_python/PyObjSnapshot.hpp | 44 ++++---- typed_python/PyPyObjGraphSnapshot.cpp | 138 ++++++++++++++++++++++++++ typed_python/PyPyObjGraphSnapshot.hpp | 40 ++++++++ typed_python/PyPyObjSnapshot.cpp | 61 ++++++++---- typed_python/PyPyObjSnapshot.hpp | 7 +- typed_python/_types.cpp | 6 ++ typed_python/all.cpp | 2 + typed_python/py_obj_snapshot_test.py | 121 ++++++++++++---------- 10 files changed, 419 insertions(+), 95 deletions(-) create mode 100644 typed_python/PyObjGraphSnapshot.hpp create mode 100644 typed_python/PyObjSnapshot.cpp create mode 100644 typed_python/PyPyObjGraphSnapshot.cpp create mode 100644 typed_python/PyPyObjGraphSnapshot.hpp diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp new file mode 100644 index 000000000..98a17e37a --- /dev/null +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -0,0 +1,65 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include + + +class PyObjSnapshot; + + +/********************************* +PyObjGraphSnapshot + +Holds a collection of PyObjSnapshot objects. When this dies, they'll die. + +There's a global PyObjGraphSnapshot that holds all the interned objects. + +**********************************/ + +class PyObjGraphSnapshot { +public: + PyObjGraphSnapshot() {} + + ~PyObjGraphSnapshot() { + for (auto oPtr: mObjects) { + delete oPtr; + } + } + + // get the "internal" graph snapshot, which is responsible for holding all the objects + // that are actually interned inside the system. + static PyObjGraphSnapshot& internal() { + static PyObjGraphSnapshot* graph = new PyObjGraphSnapshot(); + return *graph; + } + + void registerSnapshot(PyObjSnapshot* obj) { + if (obj->getGraph() != this) { + throw std::runtime_error("Can't register a snapshot for a different graph"); + } + + mObjects.insert(obj); + } + + const std::unordered_set& getObjects() const { + return mObjects; + } + +private: + std::unordered_set mObjects; +}; diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp new file mode 100644 index 000000000..317e62703 --- /dev/null +++ b/typed_python/PyObjSnapshot.cpp @@ -0,0 +1,30 @@ +#include "PyObjSnapshot.hpp" +#include "PyObjGraphSnapshot.hpp" + + +/* static */ +PyObjSnapshot* PyObjSnapshot::internalizePyObj( + PyObject* val, + std::unordered_map& constantMapCache, + const std::map<::Type*, ::Type*>& groupMap, + bool linkBackToOriginalObject, + PyObjGraphSnapshot* graph +) { + auto it = constantMapCache.find(val); + + if (it != constantMapCache.end()) { + return it->second; + } + + constantMapCache[val] = new PyObjSnapshot(graph); + + if (graph) { + graph->registerSnapshot(constantMapCache[val]); + } + + constantMapCache[val]->becomeInternalizedOf( + val, constantMapCache, groupMap, linkBackToOriginalObject, graph + ); + + return constantMapCache[val]; +} diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index af0eb36e0..31203e194 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -21,6 +21,8 @@ #include "Instance.hpp" #include "PythonTypeInternals.hpp" +class PyObjGraphSnapshot; + /********************************* PyObjSnapshot @@ -39,6 +41,7 @@ changing underneath us. **********************************/ class PyObjSnapshot { +private: enum class Kind { // this should never be visible in a running program Uninitialized = 0, @@ -92,14 +95,19 @@ class PyObjSnapshot { ArbitraryPyObject }; - PyObjSnapshot() : + PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr) : mKind(Kind::Uninitialized), mType(nullptr), - mPyObject(nullptr) + mPyObject(nullptr), + mGraph(inGraph) { } public: + ~PyObjSnapshot() { + decref(mPyObject); + } + static PyObjSnapshot* ForType(Type* t) { PyObjSnapshot* res = new PyObjSnapshot(); res->mKind = Kind::Type; @@ -107,6 +115,10 @@ class PyObjSnapshot { return res; } + PyObjGraphSnapshot* getGraph() const { + return mGraph; + } + bool isUninitialized() const { return mKind == Kind::Uninitialized; } @@ -284,28 +296,13 @@ class PyObjSnapshot { return PyUnicode_AsUTF8(o); } - // return a CVPO for 'val', stashing it in 'constantMapCache' - // in case we hit a recursion. static PyObjSnapshot* internalizePyObj( PyObject* val, std::unordered_map& constantMapCache, const std::map<::Type*, ::Type*>& groupMap, - bool linkBackToOriginalObject = true - ) { - auto it = constantMapCache.find(val); - - if (it != constantMapCache.end()) { - return it->second; - } - - constantMapCache[val] = new PyObjSnapshot(); - - constantMapCache[val]->becomeInternalizedOf( - val, constantMapCache, groupMap, linkBackToOriginalObject - ); - - return constantMapCache[val]; - } + bool linkBackToOriginalObject=true, + PyObjGraphSnapshot* graph=nullptr + ); static PyTypeObject* createAVanillaType() { PyObjectStealer emptyBases(PyTuple_New(0)); @@ -370,7 +367,8 @@ class PyObjSnapshot { PyObject* val, std::unordered_map& constantMapCache, const std::map<::Type*, ::Type*>& groupMap, - bool linkBackToOriginalObject + bool linkBackToOriginalObject, + PyObjGraphSnapshot* graph ) { // we're always the internalized version of this object if (linkBackToOriginalObject) { @@ -379,7 +377,7 @@ class PyObjSnapshot { auto internalize = [&](PyObject* o) { return PyObjSnapshot::internalizePyObj( - o, constantMapCache, groupMap, linkBackToOriginalObject + o, constantMapCache, groupMap, linkBackToOriginalObject, graph ); }; @@ -1309,7 +1307,9 @@ class PyObjSnapshot { } } } + Kind mKind; + PyObjGraphSnapshot* mGraph; ::Type* mType; ::Instance mInstance; diff --git a/typed_python/PyPyObjGraphSnapshot.cpp b/typed_python/PyPyObjGraphSnapshot.cpp new file mode 100644 index 000000000..5f49318f3 --- /dev/null +++ b/typed_python/PyPyObjGraphSnapshot.cpp @@ -0,0 +1,138 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyPyObjGraphSnapshot.hpp" + + +PyDoc_STRVAR(PyPyObjGraphSnapshot_doc, + "A snapshot of a collection of python objects.\n\n" +); + +PyMethodDef PyPyObjGraphSnapshot_methods[] = { + {NULL} /* Sentinel */ +}; + + +/* static */ +void PyPyObjGraphSnapshot::dealloc(PyPyObjGraphSnapshot *self) +{ + if (self->mOwnsSnapshot) { + delete self->mGraphSnapshot; + } + + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* PyPyObjGraphSnapshot::newPyObjGraphSnapshot(PyObjGraphSnapshot* g, bool ownsIt) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)PyType_PyObjGraphSnapshot.tp_alloc(&PyType_PyObjGraphSnapshot, 0); + self->mGraphSnapshot = g; + self->mOwnsSnapshot = ownsIt; + + return (PyObject*)self; +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyPyObjGraphSnapshot* self; + + self = (PyPyObjGraphSnapshot*)type->tp_alloc(type, 0); + + if (self != NULL) { + self->mGraphSnapshot = nullptr; + self->mOwnsSnapshot = false; + } + + return (PyObject*)self; +} + +PyObject* PyPyObjGraphSnapshot::tp_repr(PyObject *selfObj) { + PyPyObjGraphSnapshot* pyGraph = (PyPyObjGraphSnapshot*)selfObj; + + return translateExceptionToPyObject([&]() { + return PyUnicode_FromString( + ( + "PyObjGraphSnapshot(" + + format(pyGraph->mGraphSnapshot->getObjects().size()) + + ")" + ).c_str() + ); + }); +} + + +/* static */ +int PyPyObjGraphSnapshot::init(PyPyObjGraphSnapshot *self, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_RuntimeError, "PyObjGraphSnapshot cannot be initialized directly"); + return -1; +} + + +PyTypeObject PyType_PyObjGraphSnapshot = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "PyObjGraph2Snapshot", + .tp_basicsize = sizeof(PyPyObjGraphSnapshot), + .tp_itemsize = 0, + .tp_dealloc = (destructor) PyPyObjGraphSnapshot::dealloc, + #if PY_MINOR_VERSION < 8 + .tp_print = 0, + #else + .tp_vectorcall_offset = 0, // printfunc (Changed to tp_vectorcall_offset in Python 3.8) + #endif + .tp_getattr = 0, + .tp_setattr = 0, + .tp_as_async = 0, + .tp_repr = PyPyObjGraphSnapshot::tp_repr, + .tp_as_number = 0, + .tp_as_sequence = 0, + .tp_as_mapping = 0, + .tp_hash = 0, + .tp_call = 0, + .tp_str = 0, + .tp_getattro = 0, + .tp_setattro = 0, + .tp_as_buffer = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = PyPyObjGraphSnapshot_doc, + .tp_traverse = 0, + .tp_clear = 0, + .tp_richcompare = 0, + .tp_weaklistoffset = 0, + .tp_iter = 0, + .tp_iternext = 0, + .tp_methods = PyPyObjGraphSnapshot_methods, + .tp_members = 0, + .tp_getset = 0, + .tp_base = 0, + .tp_dict = 0, + .tp_descr_get = 0, + .tp_descr_set = 0, + .tp_dictoffset = 0, + .tp_init = (initproc) PyPyObjGraphSnapshot::init, + .tp_alloc = 0, + .tp_new = PyPyObjGraphSnapshot::new_, + .tp_free = 0, + .tp_is_gc = 0, + .tp_bases = 0, + .tp_mro = 0, + .tp_cache = 0, + .tp_subclasses = 0, + .tp_weaklist = 0, + .tp_del = 0, + .tp_version_tag = 0, + .tp_finalize = 0, +}; diff --git a/typed_python/PyPyObjGraphSnapshot.hpp b/typed_python/PyPyObjGraphSnapshot.hpp new file mode 100644 index 000000000..7dd773d25 --- /dev/null +++ b/typed_python/PyPyObjGraphSnapshot.hpp @@ -0,0 +1,40 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyInstance.hpp" +#include "PyObjGraphSnapshot.hpp" + +class PyPyObjGraphSnapshot { +public: + PyObject_HEAD + + PyObjGraphSnapshot* mGraphSnapshot; + bool mOwnsSnapshot; + + static PyObject* tp_repr(PyObject *selfObj); + + static void dealloc(PyPyObjGraphSnapshot *self); + + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); + + static PyObject* newPyObjGraphSnapshot(PyObjGraphSnapshot* o, bool ownsIt); + + static int init(PyPyObjGraphSnapshot *self, PyObject *args, PyObject *kwargs); +}; + +extern PyTypeObject PyType_PyObjGraphSnapshot; diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index 82418caf7..55d21b93e 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -15,10 +15,11 @@ ******************************************************************************/ #include "PyPyObjSnapshot.hpp" - +#include "PyPyObjGraphSnapshot.hpp" +#include "PyObjGraphSnapshot.hpp" PyDoc_STRVAR(PyPyObjSnapshot_doc, - "A single overload of a Function type object.\n\n" + "A single object in a snapshot graph.\n\n" ); PyMethodDef PyPyObjSnapshot_methods[] = { @@ -41,14 +42,18 @@ PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwar std::unordered_map constantMapCache; std::map<::Type*, ::Type*> groupMap; + PyObjGraphSnapshot* graph = new PyObjGraphSnapshot(); + PyObjSnapshot* object = PyObjSnapshot::internalizePyObj( instance, constantMapCache, groupMap, - linkBack + linkBack, + graph ); - return PyPyObjSnapshot::newPyObjSnapshot(object); + PyObjectStealer graphObj(PyPyObjGraphSnapshot::newPyObjGraphSnapshot(graph, true)); + return PyPyObjSnapshot::newPyObjSnapshot(object, graphObj); }); } @@ -57,25 +62,20 @@ void PyPyObjSnapshot::dealloc(PyPyObjSnapshot *self) { decref(self->mElements); decref(self->mKeys); + decref(self->mGraph); + decref(self->mByKey); Py_TYPE(self)->tp_free((PyObject*)self); } -PyObject* PyPyObjSnapshot::newPyObjSnapshot(PyObjSnapshot* g) { - static std::unordered_map memo; - - auto it = memo.find(g); - if (it != memo.end()) { - return incref(it->second); - } - +PyObject* PyPyObjSnapshot::newPyObjSnapshot(PyObjSnapshot* g, PyObject* pyGraphPtr) { PyPyObjSnapshot* self = (PyPyObjSnapshot*)PyType_PyObjSnapshot.tp_alloc(&PyType_PyObjSnapshot, 0); + self->mPyobj = g; + self->mGraph = incref(pyGraphPtr); self->mElements = nullptr; self->mKeys = nullptr; self->mByKey = nullptr; - memo[g] = incref((PyObject*)self); - return (PyObject*)self; } @@ -90,6 +90,7 @@ PyObject* PyPyObjSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kw self->mPyobj = nullptr; self->mElements = nullptr; self->mKeys = nullptr; + self->mGraph = nullptr; self->mByKey = nullptr; } @@ -110,7 +111,10 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { auto it = obj->namedElements().find(attr); if (it != obj->namedElements().end()) { - return PyPyObjSnapshot::newPyObjSnapshot(it->second); + return PyPyObjSnapshot::newPyObjSnapshot( + it->second, + it->second->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + ); } auto it2 = obj->namedInts().find(attr); @@ -130,6 +134,10 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { return PyUnicode_FromString(obj->kindAsString().c_str()); } + if (attr == "graph") { + return incref(pyCVPO->mGraph ? pyCVPO->mGraph : Py_None); + } + if (attr == "type" && obj->getType()) { return incref((PyObject*)PyInstance::typeObj(obj->getType())); } @@ -154,7 +162,13 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { if (!pyCVPO->mElements) { pyCVPO->mElements = PyList_New(0); for (auto p: obj->elements()) { - PyList_Append(pyCVPO->mElements, PyPyObjSnapshot::newPyObjSnapshot(p)); + PyList_Append( + pyCVPO->mElements, + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + ) + ); } } @@ -165,7 +179,13 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { if (!pyCVPO->mKeys) { pyCVPO->mKeys = PyList_New(0); for (auto p: obj->keys()) { - PyList_Append(pyCVPO->mKeys, PyPyObjSnapshot::newPyObjSnapshot(p)); + PyList_Append( + pyCVPO->mKeys, + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + ) + ); } } @@ -176,10 +196,15 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { if (!pyCVPO->mByKey) { pyCVPO->mByKey = PyDict_New(); for (long k = 0; k < obj->keys().size(); k++) { + PyObjSnapshot* p = obj->elements()[k]; + PyDict_SetItem( pyCVPO->mByKey, obj->keys()[k]->getPyObj(), - PyPyObjSnapshot::newPyObjSnapshot(obj->elements()[k]) + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + ) ); } } diff --git a/typed_python/PyPyObjSnapshot.hpp b/typed_python/PyPyObjSnapshot.hpp index d643f6c08..1c4a78a74 100644 --- a/typed_python/PyPyObjSnapshot.hpp +++ b/typed_python/PyPyObjSnapshot.hpp @@ -23,6 +23,11 @@ class PyPyObjSnapshot { PyObject_HEAD PyObjSnapshot* mPyobj; + + // a graph object we're keeping alive + PyObject* mGraph; + + // caches of our 'elements/keys/byKey' members, which we produce on demand PyObject* mElements; PyObject* mKeys; PyObject* mByKey; @@ -37,7 +42,7 @@ class PyPyObjSnapshot { static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); - static PyObject* newPyObjSnapshot(PyObjSnapshot* o); + static PyObject* newPyObjSnapshot(PyObjSnapshot* o, PyObject* pyGraphPtr=nullptr); static int init(PyPyObjSnapshot *self, PyObject *args, PyObject *kwargs); }; diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index a059bd98b..5c37d789a 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -38,6 +38,7 @@ #include "PyFunctionOverload.hpp" #include "PyFunctionGlobal.hpp" #include "PyPyObjSnapshot.hpp" +#include "PyPyObjGraphSnapshot.hpp" #include "PyModuleRepresentation.hpp" #include "_types.hpp" #include "CompilerVisibleObjectVisitor.hpp" @@ -3863,8 +3864,13 @@ PyInit__types(void) return NULL; } + if (PyType_Ready(&PyType_PyObjGraphSnapshot) < 0) { + return NULL; + } + PyModule_AddObject(module, "FunctionOverload", (PyObject*)incref(&PyType_FunctionOverload)); PyModule_AddObject(module, "PyObjSnapshot", (PyObject*)incref(&PyType_PyObjSnapshot)); + PyModule_AddObject(module, "PyObjGraphSnapshot", (PyObject*)incref(&PyType_PyObjGraphSnapshot)); PyModule_AddObject(module, "FunctionGlobal", (PyObject*)incref(&PyType_FunctionGlobal)); PyModule_AddObject(module, "Slab", (PyObject*)incref(&PyType_Slab)); PyModule_AddObject(module, "ModuleRepresentation", (PyObject*)incref(&PyType_ModuleRepresentation)); diff --git a/typed_python/all.cpp b/typed_python/all.cpp index 842bdcb47..7538e7c99 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -78,7 +78,9 @@ compile the entire group all at once. #include "PyTemporaryReferenceTracer.cpp" #include "PyFunctionOverload.cpp" #include "PyFunctionGlobal.cpp" +#include "PyObjSnapshot.cpp" #include "PyPyObjSnapshot.cpp" +#include "PyPyObjGraphSnapshot.cpp" #include "lz4.c" #include "lz4frame.c" diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index 0eac66a6d..82e58254b 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -4,7 +4,7 @@ from typed_python._types import PyObjSnapshot -def test_make_cvpo_basic(): +def test_make_snapshot_basic(): assert PyObjSnapshot.create(int).kind == 'Type' assert PyObjSnapshot.create((1, 2, 3)).kind == 'PyTuple' @@ -30,47 +30,47 @@ def test_make_cvpo_basic(): assert PyObjSnapshot.create(ListOf(int)((1, 2, 3))).instance[1] == 2 -def test_cvpo_function(): +def test_snapshot_function(): y = 10 def f(x: int, z=20): return 10 + y - cvpo = PyObjSnapshot.create(f) - - assert cvpo.kind == 'PyFunction' - assert cvpo.name == 'f' - assert cvpo.moduleName == 'typed_python.py_obj_snapshot_test' - assert cvpo.func_name.kind == 'String' - assert cvpo.func_module.kind == 'String' - assert cvpo.func_closure.kind == 'PyTuple' - assert cvpo.func_closure.elements[0].kind == 'PyCell' - assert cvpo.func_closure.elements[0].cell_contents.pyobj == 10 - assert cvpo.func_globals.kind == "PyModuleDict" - assert cvpo.func_globals.name == 'typed_python.py_obj_snapshot_test' - assert cvpo.func_annotations.pyobj['x'] is int - assert cvpo.func_defaults.pyobj == (20,) - assert not hasattr(cvpo, 'func_kwdefaults') - - assert cvpo.func_code.kind == 'PyCodeObject' - assert cvpo.func_code.co_argcount == 2 - assert cvpo.func_code.co_kwonlyargcount == 0 - assert cvpo.func_code.co_flags == f.__code__.co_flags - assert cvpo.func_code.co_posonlyargcount == f.__code__.co_posonlyargcount - assert cvpo.func_code.co_nlocals == f.__code__.co_nlocals - assert cvpo.func_code.co_stacksize == f.__code__.co_stacksize - assert cvpo.func_code.co_firstlineno == f.__code__.co_firstlineno - assert cvpo.func_code.co_code.pyobj == f.__code__.co_code - assert cvpo.func_code.co_consts.pyobj == f.__code__.co_consts - assert cvpo.func_code.co_names.pyobj == f.__code__.co_names - assert cvpo.func_code.co_varnames.pyobj == f.__code__.co_varnames - assert cvpo.func_code.co_freevars.pyobj == f.__code__.co_freevars - assert cvpo.func_code.co_cellvars.pyobj == f.__code__.co_cellvars - assert cvpo.func_code.co_name.pyobj == f.__code__.co_name - assert cvpo.func_code.co_filename.pyobj == f.__code__.co_filename - - -def test_cvpo_numpy_internals(): + snapshot = PyObjSnapshot.create(f) + + assert snapshot.kind == 'PyFunction' + assert snapshot.name == 'f' + assert snapshot.moduleName == 'typed_python.py_obj_snapshot_test' + assert snapshot.func_name.kind == 'String' + assert snapshot.func_module.kind == 'String' + assert snapshot.func_closure.kind == 'PyTuple' + assert snapshot.func_closure.elements[0].kind == 'PyCell' + assert snapshot.func_closure.elements[0].cell_contents.pyobj == 10 + assert snapshot.func_globals.kind == "PyModuleDict" + assert snapshot.func_globals.name == 'typed_python.py_obj_snapshot_test' + assert snapshot.func_annotations.pyobj['x'] is int + assert snapshot.func_defaults.pyobj == (20,) + assert not hasattr(snapshot, 'func_kwdefaults') + + assert snapshot.func_code.kind == 'PyCodeObject' + assert snapshot.func_code.co_argcount == 2 + assert snapshot.func_code.co_kwonlyargcount == 0 + assert snapshot.func_code.co_flags == f.__code__.co_flags + assert snapshot.func_code.co_posonlyargcount == f.__code__.co_posonlyargcount + assert snapshot.func_code.co_nlocals == f.__code__.co_nlocals + assert snapshot.func_code.co_stacksize == f.__code__.co_stacksize + assert snapshot.func_code.co_firstlineno == f.__code__.co_firstlineno + assert snapshot.func_code.co_code.pyobj == f.__code__.co_code + assert snapshot.func_code.co_consts.pyobj == f.__code__.co_consts + assert snapshot.func_code.co_names.pyobj == f.__code__.co_names + assert snapshot.func_code.co_varnames.pyobj == f.__code__.co_varnames + assert snapshot.func_code.co_freevars.pyobj == f.__code__.co_freevars + assert snapshot.func_code.co_cellvars.pyobj == f.__code__.co_cellvars + assert snapshot.func_code.co_name.pyobj == f.__code__.co_name + assert snapshot.func_code.co_filename.pyobj == f.__code__.co_filename + + +def test_snapshot_numpy_internals(): assert PyObjSnapshot.create(numpy.array).kind == 'NamedPyObject' # in theory, we could do better than this by using the reduce pathway. @@ -83,7 +83,7 @@ def test_cvpo_numpy_internals(): assert PyObjSnapshot.create(numpy.array([1])).kind == 'ArbitraryPyObject' -def test_cvpo_builtins(): +def test_snapshot_builtins(): assert PyObjSnapshot.create(__builtins__).kind == 'PyModuleDict' assert PyObjSnapshot.create(__builtins__).module_dict_of.kind == 'PyModule' assert PyObjSnapshot.create(set).kind == 'NamedPyObject' @@ -92,7 +92,7 @@ def test_cvpo_builtins(): assert PyObjSnapshot.create(range).kind == 'NamedPyObject' -def test_cvpo_class(): +def test_snapshot_class(): class C: def f(self): return gInst @@ -119,24 +119,24 @@ def clsMeth(cls): gInst = G() gMeth = gInst.f - cvpo = PyObjSnapshot.create(C) + snapshot = PyObjSnapshot.create(C) - assert cvpo.kind == 'PyClass' - assert cvpo.name == 'C' - assert cvpo.moduleName == 'typed_python.py_obj_snapshot_test' - assert cvpo.cls_dict.kind == 'PyClassDict' + assert snapshot.kind == 'PyClass' + assert snapshot.name == 'C' + assert snapshot.moduleName == 'typed_python.py_obj_snapshot_test' + assert snapshot.cls_dict.kind == 'PyClassDict' - assert cvpo.cls_dict.byKey['aProp'].kind == 'PyProperty' - assert cvpo.cls_dict.byKey['aProp'].prop_get.kind == 'PyFunction' + assert snapshot.cls_dict.byKey['aProp'].kind == 'PyProperty' + assert snapshot.cls_dict.byKey['aProp'].prop_get.kind == 'PyFunction' - assert cvpo.cls_dict.byKey['f'].kind == 'PyFunction' + assert snapshot.cls_dict.byKey['f'].kind == 'PyFunction' - gMethPO = cvpo.cls_dict.byKey['fMeth'].func_closure.elements[0].cell_contents + gMethPO = snapshot.cls_dict.byKey['fMeth'].func_closure.elements[0].cell_contents assert gMethPO.kind == 'PyBoundMethod' assert gMethPO.meth_self.pyobj is gInst assert gMethPO.meth_func.kind == 'PyFunction' - gInstPO = cvpo.cls_dict.byKey['f'].func_closure.elements[0].cell_contents + gInstPO = snapshot.cls_dict.byKey['f'].func_closure.elements[0].cell_contents assert gInstPO.pyobj is gInst assert gInstPO.inst_dict.kind == 'PyDict' gInstPO.inst_dict.byKey['y'].kind == 'Instance' @@ -156,7 +156,7 @@ def clsMeth(cls): assert GPO.cls_dict.byKey['clsMeth'].meth_func.kind == 'PyFunction' -def test_cvpo_rehydration(): +def test_snapshot_rehydration(): assert PyObjSnapshot.create(1, False).pyobj == 1 assert PyObjSnapshot.create('1', False).pyobj == '1' assert PyObjSnapshot.create(b'1', False).pyobj == b'1' @@ -193,17 +193,30 @@ def getCInstMeth(self): cInst = C() cInstMeth = cInst.getCInstMeth - C2cvpo = PyObjSnapshot.create(C, False) - C2 = C2cvpo.pyobj + C2snapshot = PyObjSnapshot.create(C, False) + C2 = C2snapshot.pyobj assert C2 is not C assert C2.__name__ == C.__name__ assert C2.__module__ == C.__module__ assert C2.getCInst is not C.getCInst - c2Inst = C2cvpo.cls_dict.byKey['getCInst'].func_closure.elements[0].cell_contents.pyobj - c2InstMeth = C2cvpo.cls_dict.byKey['getCInstMeth'].func_closure.elements[0].cell_contents.pyobj + c2Inst = C2snapshot.cls_dict.byKey['getCInst'].func_closure.elements[0].cell_contents.pyobj + c2InstMeth = C2snapshot.cls_dict.byKey['getCInstMeth'].func_closure.elements[0].cell_contents.pyobj assert C2().getCInst() is c2Inst assert C2().getCInstMeth() is c2InstMeth assert c2InstMeth() is c2InstMeth + + +def test_snapshot_graph(): + def f(): + return f + + s1 = PyObjSnapshot.create(f) + s2 = PyObjSnapshot.create(f) + + assert s1.graph is not s2.graph + assert s1.kind == 'PyFunction' + assert s1.func_closure.graph is s1.graph + assert s1.func_closure.graph is not s2.func_closure.graph From 144aca62b0dfea805df87c9dc7914f132418979a Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sat, 29 Jul 2023 21:02:02 +0000 Subject: [PATCH 65/83] PyObjSnapshot can model all TP types now. --- typed_python/FunctionGlobal.hpp | 18 +- typed_python/FunctionOverload.hpp | 10 +- typed_python/FunctionType.hpp | 7 +- typed_python/Instance.cpp | 41 ++ typed_python/Instance.hpp | 39 ++ typed_python/PyObjSnapshot.cpp | 998 ++++++++++++++++++++++++++- typed_python/PyObjSnapshot.hpp | 706 ++++++++----------- typed_python/PyPyObjSnapshot.cpp | 14 +- typed_python/Type.cpp | 16 +- typed_python/Type.hpp | 4 + typed_python/py_obj_snapshot_test.py | 2 +- 11 files changed, 1399 insertions(+), 456 deletions(-) diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index 154787059..ecb8f4d6f 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -322,18 +322,16 @@ class FunctionGlobal { } } - FunctionGlobal withConstantsInternalized( - std::unordered_map& constantMapCache, - const std::map& groupMap - ) { + FunctionGlobal withConstantsInternalized(PyObjSnapshotMaker& maker) { if (!(isGlobalInDict() || isGlobalInCell())) { return *this; } Type* t = getValueAsType(); + if (t) { - auto it = groupMap.find(t); - if (it != groupMap.end()) { + auto it = maker.getGroupMap().find(t); + if (it != maker.getGroupMap().end()) { return FunctionGlobal::Constant( PyObjSnapshot::ForType(it->second) ); @@ -350,13 +348,7 @@ class FunctionGlobal { return FunctionGlobal::Unbound(); } - return FunctionGlobal::Constant( - PyObjSnapshot::internalizePyObj( - value, - constantMapCache, - groupMap - ) - ); + return FunctionGlobal::Constant(maker.internalize(value)); } FunctionGlobal withUpdatedInternalTypePointers(const std::map& groupMap) { diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 466c77ba0..cb89d4122 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -277,15 +277,9 @@ class FunctionOverload { } } - void internalizeConstants( - std::unordered_map& constantMapCache, - const std::map& groupMap - ) { + void internalizeConstants(PyObjSnapshotMaker& snapMaker) { for (auto& nameAndGlobal: mGlobals) { - nameAndGlobal.second = nameAndGlobal.second.withConstantsInternalized( - constantMapCache, - groupMap - ); + nameAndGlobal.second = nameAndGlobal.second.withConstantsInternalized(snapMaker); } } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index c91e9196f..d529b613f 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -160,12 +160,9 @@ class Function : public Type { // replace any direct references to PyObject we're holding internally // with PyObjSnapshot references instead - void internalizeConstants( - std::unordered_map& constantMapCache, - const std::map& groupMap - ) { + void internalizeConstants(PyObjSnapshotMaker& snapMaker) { for (auto& o: mOverloads) { - o.internalizeConstants(constantMapCache, groupMap); + o.internalizeConstants(snapMaker); } } diff --git a/typed_python/Instance.cpp b/typed_python/Instance.cpp index 27ecdb4c2..a418c5429 100644 --- a/typed_python/Instance.cpp +++ b/typed_python/Instance.cpp @@ -78,6 +78,31 @@ Instance::Instance(const Instance& other) : mLayout(other.mLayout) { } } +Instance::Instance(const InstanceRef& other) { + Type* t = other.type(); + instance_ptr p = other.data(); + + if (t->isNone()) { + return; + } + + t->assertForwardsResolvedSufficientlyToInstantiate(); + + layout* l = (layout*)tp_malloc(sizeof(layout) + t->bytecount()); + + try { + t->copy_constructor(l->data, p); + } catch(...) { + tp_free(l); + throw; + } + + l->refcount = 1; + l->type = t; + + mLayout = l; +} + Instance::Instance(instance_ptr p, Type* t) : mLayout(nullptr) { if (t->isNone()) { return; @@ -167,7 +192,23 @@ typed_python_hash_type Instance::hash() const { return mLayout->type->hash(mLayout->data); } +PyObject* Instance::extractPyobj() const { + return ref().extractPyobj(); +} + Type* Instance::extractType(bool includePrimitives) const { + return ref().extractType(includePrimitives); +} + +PyObject* InstanceRef::extractPyobj() const { + if (type()->isPythonObjectOfType()) { + return PyObjectHandleTypeBase::getPyObj(data()); + } + + return nullptr; +} + +Type* InstanceRef::extractType(bool includePrimitives) const { if ( type()->isPythonObjectOfType() && PyType_Check( diff --git a/typed_python/Instance.hpp b/typed_python/Instance.hpp index fb70fd3d2..8dc41bcd8 100644 --- a/typed_python/Instance.hpp +++ b/typed_python/Instance.hpp @@ -53,6 +53,23 @@ class InstanceRef { return mData; } + bool operator<(const InstanceRef& other) const { + if (mType < other.mType) { return true; } + if (mType > other.mType) { return false; } + if (mData < other.mData) { return true; } + return false; + } + + bool operator==(const InstanceRef& other) const { + return mType == other.mType && mData == other.mData; + } + + // if we wrap a Type* as a python object, return it. Otherwise nullptr. + Type* extractType(bool includePrimitives=false) const; + + // if we wrap a PyObject return it. Otherwise nullptr. + PyObject* extractPyobj() const; + private: instance_ptr mData; Type* mType; @@ -91,6 +108,12 @@ class Instance { Instance(instance_ptr p, Type* t); + Instance(const InstanceRef& ref); + + InstanceRef ref() const { + return InstanceRef(data(), type()); + } + template static Instance createAndInitialize(Type* t, const initializer_type& initFun) { return Instance(t, initFun); @@ -145,6 +168,9 @@ class Instance { // if we wrap a Type* as a python object, return it. Otherwise nullptr. Type* extractType(bool includePrimitives=false) const; + // if we wrap a PyObject return it. Otherwise nullptr. + PyObject* extractPyobj() const; + template T& cast() { return *(T*)data(); @@ -170,3 +196,16 @@ class Instance { // if the nullptr, then this is the None object. layout* mLayout; }; + +namespace std { + template<> + struct hash { + typedef InstanceRef argument_type; + typedef std::size_t result_type; + + result_type operator()(argument_type const& s) const noexcept { + return std::hash()((void*)s.data()) ^ std::hash()((void*)s.type()); + } + }; +} + diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 317e62703..a0d637ad4 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -2,29 +2,999 @@ #include "PyObjGraphSnapshot.hpp" +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionArg& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBinding& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBindingStep& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const MemberDefinition& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionGlobal& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(PyObject* val) +{ + Type* t = PyInstance::extractTypeFrom(val, true); + if (t) { + return internalize(t); + } + + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); + if (instanceType) { + return internalize( + InstanceRef( + ((PyInstance*)val)->dataPtr(), + instanceType + ) + ); + } + + auto it = mObjMapCache.find(val); + + if (it != mObjMapCache.end()) { + return it->second; + } + + mObjMapCache[val] = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(mObjMapCache[val]); + } + + mObjMapCache[val]->becomeInternalizedOf(val, *this); + + return mObjMapCache[val]; +} + /* static */ -PyObjSnapshot* PyObjSnapshot::internalizePyObj( +PyObjSnapshot* PyObjSnapshotMaker::internalize(Type* val) +{ + // TODO: this doesn't really belong here. This infra is more general. + if (mGroupMap.find(val) != mGroupMap.end()) { + val = mGroupMap.find(val)->second; + } else { + if (val->isForwardDefined()) { + if (val->isResolved()) { + val = val->forwardResolvesTo(); + } + } + } + + auto it = mTypeMapCache.find(val); + + if (it != mTypeMapCache.end()) { + return it->second; + } + + mTypeMapCache[val] = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(mTypeMapCache[val]); + } + + mTypeMapCache[val]->becomeInternalizedOf(val, *this); + + return mTypeMapCache[val]; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(InstanceRef val) +{ + Type* t = val.extractType(true); + if (t) { + return internalize(t); + } + + PyObject* o = val.extractPyobj(); + if (o) { + return internalize(o); + } + + auto it = mInstanceCache.find(val); + + if (it != mInstanceCache.end()) { + return it->second; + } + + mInstanceCache[val] = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(mInstanceCache[val]); + } + + mInstanceCache[val]->becomeInternalizedOf(val, *this); + + return mInstanceCache[val]; +} + + +void PyObjSnapshot::becomeInternalizedOf( + Type* t, + PyObjSnapshotMaker& maker +) { + if (t->isListOf()) { + mKind = Kind::ListOfType; + mNamedElements["element_type"] = maker.internalize(((ListOfType*)t)->getEltType()); + return; + } + + if (t->isTupleOf()) { + mKind = Kind::TupleOfType; + mNamedElements["element_type"] = maker.internalize(((TupleOfType*)t)->getEltType()); + return; + } + + if (t->isTuple()) { + mKind = Kind::TupleType; + for (auto eltType: ((Tuple*)t)->getTypes()) { + mElements.push_back(maker.internalize(eltType)); + } + return; + } + + if (t->isNamedTuple()) { + mKind = Kind::NamedTupleType; + for (auto eltType: ((NamedTuple*)t)->getTypes()) { + mElements.push_back(maker.internalize(eltType)); + } + for (auto name: ((NamedTuple*)t)->getNames()) { + mNames.push_back(name); + } + return; + } + + if (t->isOneOf()) { + mKind = Kind::OneOfType; + for (auto eltType: ((OneOfType*)t)->getTypes()) { + mElements.push_back(maker.internalize(eltType)); + } + return; + } + + if (t->isValue()) { + mKind = Kind::ValueType; + mNamedElements["value_instance"] = maker.internalize( + ((Value*)t)->value() + ); + return; + } + + if (t->isDict()) { + mKind = Kind::DictType; + mNamedElements["key_type"] = maker.internalize( + ((DictType*)t)->keyType() + ); + mNamedElements["value_type"] = maker.internalize( + ((DictType*)t)->valueType() + ); + return; + } + + if (t->isConstDict()) { + mKind = Kind::ConstDictType; + mNamedElements["key_type"] = maker.internalize( + ((ConstDictType*)t)->keyType() + ); + mNamedElements["value_type"] = maker.internalize( + ((ConstDictType*)t)->valueType() + ); + return; + } + + if (t->isSet()) { + mKind = Kind::SetType; + mNamedElements["key_type"] = maker.internalize( + ((SetType*)t)->keyType() + ); + return; + } + + if (t->isPointerTo()) { + mKind = Kind::PointerToType; + mNamedElements["element_type"] = maker.internalize( + ((PointerTo*)t)->getEltType() + ); + return; + } + + if (t->isRefTo()) { + mKind = Kind::RefToType; + mNamedElements["element_type"] = maker.internalize( + ((RefTo*)t)->getEltType() + ); + return; + } + + if (t->isPythonObjectOfType()) { + mKind = Kind::PythonObjectOfTypeType; + mNamedElements["element_type"] = maker.internalize( + (PyObject*)((PythonObjectOfType*)t)->pyType() + ); + return; + } + + if (t->isSubclassOf()) { + mKind = Kind::SubclassOfTypeType; + mNamedElements["element_type"] = maker.internalize( + ((SubclassOfType*)t)->getSubclassOf() + ); + return; + } + + if (t->isTypedCell()) { + mKind = Kind::TypedCellType; + mNamedElements["element_type"] = maker.internalize( + ((TypedCellType*)t)->getHeldType() + ); + return; + } + + if (t->isBoundMethod()) { + mKind = Kind::BoundMethodType; + mNamedElements["element_type"] = maker.internalize( + ((BoundMethod*)t)->getFirstArgType() + ); + mNames.push_back(((BoundMethod*)t)->getFuncName()); + return; + } + + if (t->isAlternativeMatcher()) { + mKind = Kind::AlternativeMatcherType; + mNamedElements["alternative"] = maker.internalize( + ((AlternativeMatcher*)t)->getAlternative() + ); + return; + } + + if (t->isConcreteAlternative()) { + mKind = Kind::ConcreteAlternativeType; + mNamedElements["alternative"] = maker.internalize( + ((ConcreteAlternative*)t)->getAlternative() + ); + mNamedInts["which"] = ((ConcreteAlternative*)t)->which(); + return; + } + + if (t->isAlternative()) { + mKind = Kind::AlternativeType; + Alternative* alt = (Alternative*)t; + + mName = alt->name(); + mModuleName = alt->moduleName(); + + for (long k = 0; k < alt->subtypes().size(); k++) { + mElements.push_back(maker.internalize(alt->subtypes()[k].second)); + mNames.push_back(alt->subtypes()[k].first); + } + + mNamedElements["alt_methods"] = maker.internalize(alt->getMethods()); + return; + } + + if (t->isClass()) { + mKind = Kind::ClassType; + mName = ((Class*)t)->name(); + mModuleName = ((Class*)t)->moduleName(); + mNamedElements["held_class_type"] = maker.internalize(((Class*)t)->getHeldClass()); + return; + } + + if (t->isHeldClass()) { + mKind = Kind::HeldClassType; + HeldClass* cls = (HeldClass*)t; + + mName = cls->name(); + mModuleName = cls->moduleName(); + + mNamedElements["class_type"] = maker.internalize(cls->getClassType()); + mNamedElements["cls_methods"] = maker.internalize(cls->getOwnMemberFunctions()); + mNamedElements["cls_staticmethods"] = maker.internalize(cls->getOwnStaticFunctions()); + mNamedElements["cls_classmethods"] = maker.internalize(cls->getOwnClassMethods()); + mNamedElements["cls_properties"] = maker.internalize(cls->getOwnPropertyFunctions()); + mNamedElements["cls_classmembers"] = maker.internalize(cls->getOwnClassMembers()); + mNamedElements["cls_members"] = maker.internalize(cls->getOwnMembers()); + mNamedInts["cls_is_final"] = cls->isFinal(); + return; + } + + if (t->isFunction()) { + mKind = Kind::FunctionType; + Function* f = (Function*)t; + + mNamedElements["closure_type"] = maker.internalize(f->getClosureType()); + mNamedInts["is_entrypoint"] = f->isEntrypoint() ? 1 : 0; + mNamedInts["is_nocompile"] = f->isNocompile() ? 1 : 0; + + mName = f->name(); + mQualname = f->qualname(); + mModuleName = f->moduleName(); + + mNamedElements["func_overloads"] = maker.internalize(f->getOverloads()); + return; + } + + if (t->isRegister() + || t->isString() + || t->isBytes() + || t->isPyCell() + || t->isEmbeddedMessage() + || t->isNone() + ) { + mKind = Kind::PrimitiveType; + mType = t; + return; + } + + mKind = Kind::Type; + mType = t; +} + + + +void PyObjSnapshot::becomeInternalizedOf( PyObject* val, - std::unordered_map& constantMapCache, - const std::map<::Type*, ::Type*>& groupMap, - bool linkBackToOriginalObject, - PyObjGraphSnapshot* graph + PyObjSnapshotMaker& maker ) { - auto it = constantMapCache.find(val); + if (PyInstance::extractTypeFrom(val, true)) { + throw std::runtime_error("types should have been passed to the other pathway"); + } - if (it != constantMapCache.end()) { - return it->second; + // we're always the internalized version of this object + if (maker.linkBackToOriginalObject()) { + mPyObject = incref(val); + } + + PyObject* environType = staticPythonInstance("os", "_Environ"); + + if (val->ob_type == (PyTypeObject*)environType) { + mKind = Kind::NamedPyObject; + mName = "_Environ"; + mModuleName = "os"; + return; + } + + + if (PyUnicode_Check(val)) { + mKind = Kind::String; + mStringValue = PyUnicode_AsUTF8(val); + return; + } + + if (PyTuple_Check(val)) { + mKind = Kind::PyTuple; + for (long i = 0; i < PyTuple_Size(val); i++) { + mElements.push_back(maker.internalize(PyTuple_GetItem(val, i))); + } + return; + } + + if (PyList_Check(val)) { + mKind = Kind::PyList; + for (long i = 0; i < PyList_Size(val); i++) { + mElements.push_back(maker.internalize(PyList_GetItem(val, i))); + } + return; + } + + if (PySet_Check(val)) { + mKind = Kind::PySet; + iterate(val, [&](PyObject* o) { + mElements.push_back(maker.internalize(o)); + }); + return; + } + + if (PyDict_Check(val)) { + // see if this is a moduledict + PyObject* mname = PyDict_GetItemString(val, "__name__"); + if (mname && PyUnicode_Check(mname)) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + PyObjectStealer moduleObj(PyObject_GetItem(sysModuleModules, mname)); + + if (moduleObj) { + PyObjectStealer moduleObjDict(PyObject_GenericGetDict(moduleObj, nullptr)); + if (!moduleObjDict) { + PyErr_Clear(); + } else + if (moduleObjDict == val) { + mKind = Kind::PyModuleDict; + mName = PyUnicode_AsUTF8(mname); + mNamedElements["module_dict_of"] = maker.internalize(moduleObj); + return; + } + } else { + PyErr_Clear(); + } + } + + // see if this is a vanilla class dict + PyObject* dictAccessor = PyDict_GetItemString(val, "__dict__"); + if (dictAccessor && dictAccessor->ob_type == &PyGetSetDescr_Type) { + PyTypeObject* clsType = PyDescr_TYPE(dictAccessor); + if (clsType) { + if (clsType->tp_dict == val) { + if (isVanillaClassType(clsType)) { + mKind = Kind::PyClassDict; + + mNamedElements["class_dict_of"] = maker.internalize( + (PyObject*)clsType + ); + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (val && PyDict_Next(val, &pos, &key, &value)) { + if (PyUnicode_Check(key) + && PyUnicode_AsUTF8(key) != std::string("__dict__") + && PyUnicode_AsUTF8(key) != std::string("__weakref__") + ) { + mElements.push_back(maker.internalize(value)); + mKeys.push_back(maker.internalize(key)); + } + } + + return; + } + } + } + } + + // this is a vanilla dict + mKind = Kind::PyDict; + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (val && PyDict_Next(val, &pos, &key, &value)) { + mElements.push_back(maker.internalize(value)); + mKeys.push_back(maker.internalize(key)); + } + return; + } + + if (PyCell_Check(val)) { + mKind = Kind::PyCell; + + if (PyCell_Get(val)) { + mNamedElements["cell_contents"] = maker.internalize(PyCell_Get(val)); + } + + return; + } + + if (PyType_Check(val)) { + PyTypeObject* tp = (PyTypeObject*)val; + + if (isVanillaClassType(tp)) { + mKind = Kind::PyClass; + + mName = tp->tp_name; + mModuleName = dictGetStringOrEmpty(tp->tp_dict, "__module__"); + + if (tp->tp_dict) { + mNamedElements["cls_dict"] = maker.internalize(tp->tp_dict); + } + if (tp->tp_bases) { + mNamedElements["cls_bases"] = maker.internalize(tp->tp_bases); + } + + return; + } + } + + if (PyFunction_Check(val)) { + mKind = Kind::PyFunction; + + PyFunctionObject* f = (PyFunctionObject*)val; + + mName = stringOrEmpty(f->func_name); + mModuleName = stringOrEmpty(f->func_module); + + if (f->func_name) { + mNamedElements["func_name"] = maker.internalize(f->func_name); + } + if (f->func_module) { + mNamedElements["func_module"] = maker.internalize(f->func_module); + } + if (f->func_qualname) { + mNamedElements["func_qualname"] = maker.internalize(f->func_qualname); + } + if (PyFunction_GetClosure(val)) { + mNamedElements["func_closure"] = maker.internalize(PyFunction_GetClosure(val)); + } + if (PyFunction_GetCode(val)) { + mNamedElements["func_code"] = maker.internalize(PyFunction_GetCode(val)); + } + if (PyFunction_GetModule(val)) { + mNamedElements["func_module"] = maker.internalize(PyFunction_GetModule(val)); + } + if (PyFunction_GetAnnotations(val)) { + mNamedElements["func_annotations"] = maker.internalize(PyFunction_GetAnnotations(val)); + } + if (PyFunction_GetDefaults(val)) { + mNamedElements["func_defaults"] = maker.internalize(PyFunction_GetDefaults(val)); + } + if (PyFunction_GetKwDefaults(val)) { + mNamedElements["func_kwdefaults"] = maker.internalize(PyFunction_GetKwDefaults(val)); + } + if (PyFunction_GetGlobals(val)) { + mNamedElements["func_globals"] = maker.internalize(PyFunction_GetGlobals(val)); + } + return; + } + + if (PyCode_Check(val)) { + mKind = Kind::PyCodeObject; + + PyCodeObject* co = (PyCodeObject*)val; + + mNamedInts["co_argcount"] = co->co_argcount; + mNamedInts["co_kwonlyargcount"] = co->co_kwonlyargcount; + mNamedInts["co_nlocals"] = co->co_nlocals; + mNamedInts["co_stacksize"] = co->co_stacksize; + mNamedInts["co_firstlineno"] = co->co_firstlineno; + mNamedInts["co_posonlyargcount"] = co->co_posonlyargcount; + mNamedInts["co_flags"] = co->co_flags; + + mNamedElements["co_code"] = maker.internalize(co->co_code); + mNamedElements["co_consts"] = maker.internalize(co->co_consts); + mNamedElements["co_names"] = maker.internalize(co->co_names); + mNamedElements["co_varnames"] = maker.internalize(co->co_varnames); + mNamedElements["co_freevars"] = maker.internalize(co->co_freevars); + mNamedElements["co_cellvars"] = maker.internalize(co->co_cellvars); + mNamedElements["co_name"] = maker.internalize(co->co_name); + mNamedElements["co_filename"] = maker.internalize(co->co_filename); + +# if PY_MINOR_VERSION >= 10 + mNamedElements["co_linetable"] = maker.internalize(co->co_linetable); +# else + mNamedElements["co_lnotab"] = maker.internalize(co->co_lnotab); +# endif + + return; + } + + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); + if (instanceType) { + throw std::runtime_error( + "PyObjSnapshotMaker::internalize should already have cast this to an instance call" + ); + return; + } + + if (val == Py_None) { + mKind = Kind::Instance; + return; + } + + if (PyBool_Check(val)) { + mKind = Kind::Instance; + mInstance = Instance::create(val == Py_True); + return; + } + + if (PyLong_Check(val)) { + mKind = Kind::Instance; + + try { + mInstance = Instance::create((int64_t)PyLong_AsLongLong(val)); + } + catch(...) { + mInstance = Instance::create((uint64_t)PyLong_AsUnsignedLongLong(val)); + } + + return; + } + + if (PyFloat_Check(val)) { + mKind = Kind::Instance; + mInstance = Instance::create(PyFloat_AsDouble(val)); + return; } - constantMapCache[val] = new PyObjSnapshot(graph); + if (PyModule_Check(val)) { + PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); - if (graph) { - graph->registerSnapshot(constantMapCache[val]); + PyObjectStealer name(PyObject_GetAttrString(val, "__name__")); + if (name) { + if (PyUnicode_Check(name)) { + PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, name)); + if (moduleObject) { + if (moduleObject == val) { + mKind = Kind::PyModule; + mName = PyUnicode_AsUTF8(name); + return; + } + } else { + PyErr_Clear(); + } + } + } else { + PyErr_Clear(); + } } - constantMapCache[val]->becomeInternalizedOf( - val, constantMapCache, groupMap, linkBackToOriginalObject, graph + if (PyBytes_Check(val)) { + mKind = Kind::Instance; + mInstance = Instance::createAndInitialize( + BytesType::Make(), + [&](instance_ptr i) { + BytesType::Make()->constructor( + i, + PyBytes_GET_SIZE(val), + PyBytes_AsString(val) + ); + } + ); + return; + } + + if (isVanillaClassType(val->ob_type)) { + mKind = Kind::PyObject; + + mNamedElements["inst_type"] = maker.internalize((PyObject*)val->ob_type); + + PyObjectStealer dict(PyObject_GenericGetDict(val, nullptr)); + if (dict) { + mNamedElements["inst_dict"] = maker.internalize(dict); + } + return; + } + + if (val->ob_type == &PyStaticMethod_Type || val->ob_type == &PyClassMethod_Type) { + if (val->ob_type == &PyStaticMethod_Type) { + mKind = Kind::PyStaticMethod; + } else { + mKind = Kind::PyClassMethod; + } + + PyObjectStealer funcObj(PyObject_GetAttrString(val, "__func__")); + + mNamedElements["meth_func"] = maker.internalize(funcObj); + + return; + } + + if (val->ob_type == &PyProperty_Type) { + mKind = Kind::PyProperty; + + JustLikeAPropertyObject* prop = (JustLikeAPropertyObject*)val; + + if (prop->prop_get) { + mNamedElements["prop_get"] = maker.internalize(prop->prop_get); + } + if (prop->prop_set) { + mNamedElements["prop_set"] = maker.internalize(prop->prop_set); + } + if (prop->prop_del) { + mNamedElements["prop_del"] = maker.internalize(prop->prop_del); + } + if (prop->prop_doc) { + mNamedElements["prop_doc"] = maker.internalize(prop->prop_doc); + } + + #if PY_MINOR_VERSION >= 10 + if (prop->prop_name) { + mNamedElements["prop_name"] = maker.internalize(prop->prop_name); + } + #endif + + return; + } + + if (val->ob_type == &PyMethod_Type) { + mKind = Kind::PyBoundMethod; + + PyObjectStealer fself(PyObject_GetAttrString(val, "__self__")); + PyObjectStealer ffunc(PyObject_GetAttrString(val, "__func__")); + + mNamedElements["meth_self"] = maker.internalize(fself); + mNamedElements["meth_func"] = maker.internalize(ffunc); + return; + } + + if (isPyObjectGloballyIdentifiable(val)) { + mKind = Kind::NamedPyObject; + + // no checks are necessary because isPyObjectGloballyIdentifiable + // confirms that this is OK. + PyObjectStealer moduleName(PyObject_GetAttrString(val, "__module__")); + PyObjectStealer clsName(PyObject_GetAttrString(val, "__name__")); + + mModuleName = PyUnicode_AsUTF8(moduleName); + mName = PyUnicode_AsUTF8(clsName); + return; + } + + mKind = Kind::ArbitraryPyObject; + mPyObject = incref(val); +} + +void PyObjSnapshot::becomeInternalizedOf( + InstanceRef val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::Instance; + mInstance = val; +} + +void PyObjSnapshot::becomeInternalizedOf( + const MemberDefinition& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::ClassMemberDefinition; + mName = val.getName(); + mNamedElements["type"] = maker.internalize(val.getType()); + mNamedElements["defaultValue"] = maker.internalize(val.getDefaultValue()); + mNamedInts["isNonempty"] = val.getIsNonempty() ? 1 : 0; +} + +void PyObjSnapshot::becomeInternalizedOf( + const FunctionGlobal& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionGlobal; + + if (val.isUnbound()) { + return; + } + + if (val.isNamedModuleMember()) { + mModuleName = val.getModuleName(); + mName = val.getName(); + mNamedElements["moduleDict"] = maker.internalize(val.getModuleDictOrCell()); + return; + } + + if (val.isGlobalInCell()) { + mNamedElements["cell"] = maker.internalize(val.getModuleDictOrCell()); + return; + } + + if (val.isGlobalInDict()) { + mNamedElements["moduleDict"] = maker.internalize(val.getModuleDictOrCell()); + mName = val.getName(); + return; + } +} + +void PyObjSnapshot::becomeInternalizedOf( + const FunctionOverload& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionOverload; + + mNamedElements["func_globals"] = maker.internalize(val.getGlobals()); + mNamedElements["func_code"] = maker.internalize(val.getFunctionCode()); + mNamedElements["func_defaults"] = maker.internalize(val.getFunctionDefaults()); + mNamedElements["func_annotations"] = maker.internalize(val.getFunctionAnnotations()); + mNamedElements["func_args"] = maker.internalize(val.getArgs()); + mNamedElements["func_closure_varnames"] = maker.internalize( + val.getFunctionClosureVarnames() + ); + mNamedElements["func_globals_in_closure_varnames"] = maker.internalize( + val.getFunctionGlobalsInClosureVarnames() + ); + + if (val.getMethodOf()) { + mNamedElements["func_method_of"] = maker.internalize(val.getMethodOf()); + } + + if (val.getSignatureFunction()) { + mNamedElements["func_signature_func"] = maker.internalize(val.getSignatureFunction()); + } + + mNamedElements["func_closure_variable_bindings"] = maker.internalize( + val.getClosureVariableBindings() ); - return constantMapCache[val]; + if (val.getReturnType()) { + mNamedElements["func_ret_type"] = maker.internalize(val.getReturnType()); + } +} + + +void PyObjSnapshot::becomeInternalizedOf( + const FunctionArg& arg, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionArg; + + mName = arg.getName(); + if (arg.getTypeFilter()) { + mNamedElements["arg_type_filter"] = maker.internalize(arg.getTypeFilter()); + } + + if (arg.getDefaultValue()) { + mNamedElements["arg_default_value"] = maker.internalize(arg.getDefaultValue()); + } + + mNamedInts["arg_is_star_arg"] = arg.getIsStarArg() ? 1 : 0; + mNamedInts["arg_is_kwarg"] = arg.getIsKwarg() ? 1 : 0; +} + +void PyObjSnapshot::becomeInternalizedOf( + const ClosureVariableBinding& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionClosureVariableBinding; + + for (long k = 0; k < val.size(); k++) { + mElements.push_back(maker.internalize(val[k])); + } +} + +void PyObjSnapshot::becomeInternalizedOf( + const ClosureVariableBindingStep& step, + PyObjSnapshotMaker& maker +) { + mKind = Kind::FunctionClosureVariableBindingStep; + + if (step.isFunction()) { + mNamedElements["step_func"] = maker.internalize(step.getFunction()); + } + if (step.isNamedField()) { + mNames.push_back(step.getNamedField()); + } + if (step.isIndexedField()) { + mNamedInts["step_index"] = step.getIndexedField(); + } +} + +void PyObjSnapshot::becomeInternalizedOf( + const std::string& val, + PyObjSnapshotMaker& maker +) { + mKind = Kind::String; + mStringValue = val; } diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 31203e194..c0be98154 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -22,7 +22,9 @@ #include "PythonTypeInternals.hpp" class PyObjGraphSnapshot; - +class PyObjSnapshot; +class FunctionGlobal; +class FunctionOverload; /********************************* PyObjSnapshot @@ -40,8 +42,74 @@ gives us a self-consistent view of the world to compile against so we don't have changing underneath us. **********************************/ + +class PyObjSnapshotMaker { +public: + PyObjSnapshotMaker( + std::unordered_map& inObjMapCache, + std::unordered_map& inTypeMapCache, + std::unordered_map& inInstanceCache, + const std::map<::Type*, ::Type*>& inGroupMap, + PyObjGraphSnapshot* inGraph, + bool inLinkBackToOriginalObject + ) : + mObjMapCache(inObjMapCache), + mTypeMapCache(inTypeMapCache), + mInstanceCache(inInstanceCache), + mGroupMap(inGroupMap), + mGraph(inGraph), + mLinkBackToOriginalObject(inLinkBackToOriginalObject) + { + } + + PyObjSnapshot* internalize(const std::string& def); + PyObjSnapshot* internalize(const MemberDefinition& def); + PyObjSnapshot* internalize(const FunctionGlobal& def); + PyObjSnapshot* internalize(const FunctionOverload& def); + PyObjSnapshot* internalize(const FunctionArg& def); + PyObjSnapshot* internalize(const ClosureVariableBinding& def); + PyObjSnapshot* internalize(const ClosureVariableBindingStep& def); + PyObjSnapshot* internalize(const std::vector& def); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::map& inBindings); + PyObjSnapshot* internalize(const std::map& inGlobals); + PyObjSnapshot* internalize(const std::map& inMethods); + PyObjSnapshot* internalize(const std::map& inMethods); + PyObjSnapshot* internalize(const std::vector& inMethods); + PyObjSnapshot* internalize(PyObject* val); + PyObjSnapshot* internalize(Type* val); + PyObjSnapshot* internalize(const Instance& val) { + return internalize(val.ref()); + } + PyObjSnapshot* internalize(InstanceRef val); + + bool linkBackToOriginalObject() const { + return mLinkBackToOriginalObject; + } + + PyObjGraphSnapshot* graph() const { + return mGraph; + } + + const std::map<::Type*, ::Type*>& getGroupMap() const { + return mGroupMap; + } + +private: + std::unordered_map& mObjMapCache; + std::unordered_map& mTypeMapCache; + std::unordered_map& mInstanceCache; + const std::map<::Type*, ::Type*>& mGroupMap; + PyObjGraphSnapshot* mGraph; + bool mLinkBackToOriginalObject; +}; + + class PyObjSnapshot { private: + friend class PyObjSnapshotMaker; + enum class Kind { // this should never be visible in a running program Uninitialized = 0, @@ -54,10 +122,68 @@ class PyObjSnapshot { // look inside of it) NamedPyObject, // we're pointing back into a typed_python Type held in mType. + // it must be a 'leaf' type with no internals Type, - // we're pointing into a TP instance that doesn't reach a more complex object. - // It can have Type leaves in it. It will be held in 'mInstance' + // we're pointing into a TP leaf instance (a register type, an int, bytes, etc.) Instance, + // we're a primitive type (like int) + PrimitiveType, + // a TP ListOf type. The element type will be in element_type + ListOfType, + // a TP TupleOf type. The element type will be in element_type + TupleOfType, + // a TP Tuple type. The subtypes will be in mElements + TupleType, + // a TP NamedTuple type. The subtypes will be in mElements and the names in mNames + NamedTupleType, + // a TP OneOf type. The subtypes will be in mElements + OneOfType, + // a TP Value type. The instance will be in value_instance + ValueType, + // a TP DictType type. Will have key_type and value_type + DictType, + // a TP ConstDictType type. Will have key_type and value_type + ConstDictType, + // a TP ConstDictType type. Will have key_type and value_type + SetType, + // a TP PointerTo type. The element type will be in element_type + PointerToType, + // a TP RefTo type. The element type will be in element_type + RefToType, + // a TP Alternative type. + AlternativeType, + // a TP ConcreteAlternative type. + ConcreteAlternativeType, + // a TP AlternativeMatcher type. + AlternativeMatcherType, + // a TP PythonObjectOfType type. + PythonObjectOfTypeType, + // a TP SubclassOfType type. + SubclassOfTypeType, + // a TP Class type. + ClassType, + // a TP HeldClass type. + HeldClassType, + // a TP FunctionType type. + FunctionType, + // a TP FunctionOverload. + FunctionOverload, + // a TP FunctionGlobal. + FunctionGlobal, + // a TP FunctionArg. + FunctionArg, + // a TP ClosureVariableBinding. + FunctionClosureVariableBinding, + // a TP ClosureVariableBinding.Step + FunctionClosureVariableBindingStep, + // a TP MemberDefinition in a Class. + ClassMemberDefinition, + // a TP BoundMethod type + BoundMethodType, + // a TP Forward type + ForwardType, + // a TP TypedCellType + TypedCellType, // a python list, with elements in mElements PyList, // a python Dict, with values in mElements and keys in mKeys @@ -92,7 +218,10 @@ class PyObjSnapshot { // PyObject without looking inside of it, and the details of this // object are insufficient to differentiate two different Function // types that both refer to different ArbitraryPyObject instances. - ArbitraryPyObject + ArbitraryPyObject, + // a bundle of types that doesn't represent a specific object or type + // but holds collections of types. + InternalBundle }; PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr) : @@ -188,6 +317,10 @@ class PyObjSnapshot { return "Uninitialized"; } + if (mKind == Kind::InternalBundle) { + return "InternalBundle"; + } + if (mKind == Kind::String) { return "String"; } @@ -204,6 +337,106 @@ class PyObjSnapshot { return "Instance"; } + if (mKind == Kind::PrimitiveType) { + return "PrimitiveType"; + } + + if (mKind == Kind::ListOfType) { + return "ListOfType"; + } + + if (mKind == Kind::TupleOfType) { + return "TupleOfType"; + } + + if (mKind == Kind::TupleType) { + return "TupleType"; + } + + if (mKind == Kind::NamedTupleType) { + return "NamedTupleType"; + } + + if (mKind == Kind::OneOfType) { + return "OneOfType"; + } + + if (mKind == Kind::ValueType) { + return "ValueType"; + } + + if (mKind == Kind::DictType) { + return "DictType"; + } + + if (mKind == Kind::ConstDictType) { + return "ConstDictType"; + } + + if (mKind == Kind::SetType) { + return "SetType"; + } + + if (mKind == Kind::PointerToType) { + return "PointerToType"; + } + + if (mKind == Kind::RefToType) { + return "RefToType"; + } + + if (mKind == Kind::AlternativeType) { + return "AlternativeType"; + } + + if (mKind == Kind::ConcreteAlternativeType) { + return "ConcreteAlternativeType"; + } + + if (mKind == Kind::AlternativeMatcherType) { + return "AlternativeMatcherType"; + } + + if (mKind == Kind::PythonObjectOfTypeType) { + return "PythonObjectOfTypeType"; + } + + if (mKind == Kind::SubclassOfTypeType) { + return "SubclassOfTypeType"; + } + + if (mKind == Kind::ClassType) { + return "ClassType"; + } + + if (mKind == Kind::HeldClassType) { + return "HeldClassType"; + } + + if (mKind == Kind::FunctionType) { + return "FunctionType"; + } + + if (mKind == Kind::FunctionOverload) { + return "FunctionOverload"; + } + + if (mKind == Kind::FunctionGlobal) { + return "FunctionGlobal"; + } + + if (mKind == Kind::BoundMethodType) { + return "BoundMethodType"; + } + + if (mKind == Kind::ForwardType) { + return "ForwardType"; + } + + if (mKind == Kind::TypedCellType) { + return "TypedCellType"; + } + if (mKind == Kind::PyList) { return "PyList"; } @@ -272,7 +505,7 @@ class PyObjSnapshot { return "ArbitraryPyObject"; } - throw std::runtime_error("Unknown PyObjSnapshot Kind"); + throw std::runtime_error("Unknown PyObjSnapshot Kind: " + format((int)mKind)); } static std::string dictGetStringOrEmpty(PyObject* dict, const char* name) { @@ -296,14 +529,6 @@ class PyObjSnapshot { return PyUnicode_AsUTF8(o); } - static PyObjSnapshot* internalizePyObj( - PyObject* val, - std::unordered_map& constantMapCache, - const std::map<::Type*, ::Type*>& groupMap, - bool linkBackToOriginalObject=true, - PyObjGraphSnapshot* graph=nullptr - ); - static PyTypeObject* createAVanillaType() { PyObjectStealer emptyBases(PyTuple_New(0)); PyObjectStealer emptyDict(PyDict_New()); @@ -364,409 +589,54 @@ class PyObjSnapshot { } void becomeInternalizedOf( - PyObject* val, - std::unordered_map& constantMapCache, - const std::map<::Type*, ::Type*>& groupMap, - bool linkBackToOriginalObject, - PyObjGraphSnapshot* graph - ) { - // we're always the internalized version of this object - if (linkBackToOriginalObject) { - mPyObject = incref(val); - } - - auto internalize = [&](PyObject* o) { - return PyObjSnapshot::internalizePyObj( - o, constantMapCache, groupMap, linkBackToOriginalObject, graph - ); - }; - - ::Type* t = PyInstance::extractTypeFrom(val, true); - - if (t) { - if (groupMap.find(t) != groupMap.end()) { - t = groupMap.find(t)->second; - } else { - if (t->isForwardDefined()) { - if (t->isResolved()) { - t = t->forwardResolvesTo(); - } - } - } - - mKind = Kind::Type; - mType = t; - return; - } - - PyObject* environType = staticPythonInstance("os", "_Environ"); - - if (val->ob_type == (PyTypeObject*)environType) { - mKind = Kind::NamedPyObject; - mName = "_Environ"; - mModuleName = "os"; - return; - } - - - if (PyUnicode_Check(val)) { - mKind = Kind::String; - mStringValue = PyUnicode_AsUTF8(val); - return; - } - - if (PyTuple_Check(val)) { - mKind = Kind::PyTuple; - for (long i = 0; i < PyTuple_Size(val); i++) { - mElements.push_back(internalize(PyTuple_GetItem(val, i))); - } - return; - } - - if (PyList_Check(val)) { - mKind = Kind::PyList; - for (long i = 0; i < PyList_Size(val); i++) { - mElements.push_back(internalize(PyList_GetItem(val, i))); - } - return; - } - - if (PySet_Check(val)) { - mKind = Kind::PySet; - iterate(val, [&](PyObject* o) { - mElements.push_back(internalize(o)); - }); - return; - } - - if (PyDict_Check(val)) { - // see if this is a moduledict - PyObject* mname = PyDict_GetItemString(val, "__name__"); - if (mname && PyUnicode_Check(mname)) { - PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); - PyObjectStealer moduleObj(PyObject_GetItem(sysModuleModules, mname)); - - if (moduleObj) { - PyObjectStealer moduleObjDict(PyObject_GenericGetDict(moduleObj, nullptr)); - if (!moduleObjDict) { - PyErr_Clear(); - } else - if (moduleObjDict == val) { - mKind = Kind::PyModuleDict; - mName = PyUnicode_AsUTF8(mname); - mNamedElements["module_dict_of"] = internalize(moduleObj); - return; - } - } else { - PyErr_Clear(); - } - } - - // see if this is a vanilla class dict - PyObject* dictAccessor = PyDict_GetItemString(val, "__dict__"); - if (dictAccessor && dictAccessor->ob_type == &PyGetSetDescr_Type) { - PyTypeObject* clsType = PyDescr_TYPE(dictAccessor); - if (clsType) { - if (clsType->tp_dict == val) { - if (isVanillaClassType(clsType)) { - mKind = Kind::PyClassDict; - - mNamedElements["class_dict_of"] = internalize( - (PyObject*)clsType - ); - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (val && PyDict_Next(val, &pos, &key, &value)) { - if (PyUnicode_Check(key) - && PyUnicode_AsUTF8(key) != std::string("__dict__") - && PyUnicode_AsUTF8(key) != std::string("__weakref__") - ) { - mElements.push_back(internalize(value)); - mKeys.push_back(internalize(key)); - } - } - - return; - } - } - } - } - - // this is a vanilla dict - mKind = Kind::PyDict; - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (val && PyDict_Next(val, &pos, &key, &value)) { - mElements.push_back(internalize(value)); - mKeys.push_back(internalize(key)); - } - return; - } - - if (PyCell_Check(val)) { - mKind = Kind::PyCell; - - if (PyCell_Get(val)) { - mNamedElements["cell_contents"] = internalize(PyCell_Get(val)); - } - - return; - } - - if (PyType_Check(val)) { - PyTypeObject* tp = (PyTypeObject*)val; - - if (isVanillaClassType(tp)) { - mKind = Kind::PyClass; - - mName = tp->tp_name; - mModuleName = dictGetStringOrEmpty(tp->tp_dict, "__module__"); - - if (tp->tp_dict) { - mNamedElements["cls_dict"] = internalize(tp->tp_dict); - } - if (tp->tp_bases) { - mNamedElements["cls_bases"] = internalize(tp->tp_bases); - } - - return; - } - } - - if (PyFunction_Check(val)) { - mKind = Kind::PyFunction; - - PyFunctionObject* f = (PyFunctionObject*)val; - - mName = stringOrEmpty(f->func_name); - mModuleName = stringOrEmpty(f->func_module); - - if (f->func_name) { - mNamedElements["func_name"] = internalize(f->func_name); - } - if (f->func_module) { - mNamedElements["func_module"] = internalize(f->func_module); - } - if (f->func_qualname) { - mNamedElements["func_qualname"] = internalize(f->func_qualname); - } - if (PyFunction_GetClosure(val)) { - mNamedElements["func_closure"] = internalize(PyFunction_GetClosure(val)); - } - if (PyFunction_GetCode(val)) { - mNamedElements["func_code"] = internalize(PyFunction_GetCode(val)); - } - if (PyFunction_GetModule(val)) { - mNamedElements["func_module"] = internalize(PyFunction_GetModule(val)); - } - if (PyFunction_GetAnnotations(val)) { - mNamedElements["func_annotations"] = internalize(PyFunction_GetAnnotations(val)); - } - if (PyFunction_GetDefaults(val)) { - mNamedElements["func_defaults"] = internalize(PyFunction_GetDefaults(val)); - } - if (PyFunction_GetKwDefaults(val)) { - mNamedElements["func_kwdefaults"] = internalize(PyFunction_GetKwDefaults(val)); - } - if (PyFunction_GetGlobals(val)) { - mNamedElements["func_globals"] = internalize(PyFunction_GetGlobals(val)); - } - return; - } - - if (PyCode_Check(val)) { - mKind = Kind::PyCodeObject; - - PyCodeObject* co = (PyCodeObject*)val; - - mNamedInts["co_argcount"] = co->co_argcount; - mNamedInts["co_kwonlyargcount"] = co->co_kwonlyargcount; - mNamedInts["co_nlocals"] = co->co_nlocals; - mNamedInts["co_stacksize"] = co->co_stacksize; - mNamedInts["co_firstlineno"] = co->co_firstlineno; - mNamedInts["co_posonlyargcount"] = co->co_posonlyargcount; - mNamedInts["co_flags"] = co->co_flags; - - mNamedElements["co_code"] = internalize(co->co_code); - mNamedElements["co_consts"] = internalize(co->co_consts); - mNamedElements["co_names"] = internalize(co->co_names); - mNamedElements["co_varnames"] = internalize(co->co_varnames); - mNamedElements["co_freevars"] = internalize(co->co_freevars); - mNamedElements["co_cellvars"] = internalize(co->co_cellvars); - mNamedElements["co_name"] = internalize(co->co_name); - mNamedElements["co_filename"] = internalize(co->co_filename); - -# if PY_MINOR_VERSION >= 10 - mNamedElements["co_linetable"] = internalize(co->co_linetable); -# else - mNamedElements["co_lnotab"] = internalize(co->co_lnotab); -# endif - - return; - } - - ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); - if (instanceType) { - mKind = Kind::Instance; - mInstance = ::Instance::create( - instanceType, - ((PyInstance*)val)->dataPtr() - ); - return; - } - - if (val == Py_None) { - mKind = Kind::Instance; - return; - } - - if (PyBool_Check(val)) { - mKind = Kind::Instance; - mInstance = Instance::create(val == Py_True); - return; - } - - if (PyLong_Check(val)) { - mKind = Kind::Instance; - - try { - mInstance = Instance::create((int64_t)PyLong_AsLongLong(val)); - } - catch(...) { - mInstance = Instance::create((uint64_t)PyLong_AsUnsignedLongLong(val)); - } - - return; - } - - if (PyFloat_Check(val)) { - mKind = Kind::Instance; - mInstance = Instance::create(PyFloat_AsDouble(val)); - return; - } - - if (PyModule_Check(val)) { - PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); - - PyObjectStealer name(PyObject_GetAttrString(val, "__name__")); - if (name) { - if (PyUnicode_Check(name)) { - PyObjectStealer moduleObject(PyObject_GetItem(sysModuleModules, name)); - if (moduleObject) { - if (moduleObject == val) { - mKind = Kind::PyModule; - mName = PyUnicode_AsUTF8(name); - return; - } - } else { - PyErr_Clear(); - } - } - } else { - PyErr_Clear(); - } - } - - if (PyBytes_Check(val)) { - mKind = Kind::Instance; - mInstance = Instance::createAndInitialize( - BytesType::Make(), - [&](instance_ptr i) { - BytesType::Make()->constructor( - i, - PyBytes_GET_SIZE(val), - PyBytes_AsString(val) - ); - } - ); - return; - } - - if (isVanillaClassType(val->ob_type)) { - mKind = Kind::PyObject; - - mNamedElements["inst_type"] = internalize((PyObject*)val->ob_type); - - PyObjectStealer dict(PyObject_GenericGetDict(val, nullptr)); - if (dict) { - mNamedElements["inst_dict"] = internalize(dict); - } - return; - } - - if (val->ob_type == &PyStaticMethod_Type || val->ob_type == &PyClassMethod_Type) { - if (val->ob_type == &PyStaticMethod_Type) { - mKind = Kind::PyStaticMethod; - } else { - mKind = Kind::PyClassMethod; - } - - PyObjectStealer funcObj(PyObject_GetAttrString(val, "__func__")); - - mNamedElements["meth_func"] = internalize(funcObj); - - return; - } - - if (val->ob_type == &PyProperty_Type) { - mKind = Kind::PyProperty; - - JustLikeAPropertyObject* prop = (JustLikeAPropertyObject*)val; - - if (prop->prop_get) { - mNamedElements["prop_get"] = internalize(prop->prop_get); - } - if (prop->prop_set) { - mNamedElements["prop_set"] = internalize(prop->prop_set); - } - if (prop->prop_del) { - mNamedElements["prop_del"] = internalize(prop->prop_del); - } - if (prop->prop_doc) { - mNamedElements["prop_doc"] = internalize(prop->prop_doc); - } - - #if PY_MINOR_VERSION >= 10 - if (prop->prop_name) { - mNamedElements["prop_name"] = internalize(prop->prop_name); - } - #endif + const std::string& val, + PyObjSnapshotMaker& maker + ); - return; - } + void becomeInternalizedOf( + const FunctionArg& val, + PyObjSnapshotMaker& maker + ); - if (val->ob_type == &PyMethod_Type) { - mKind = Kind::PyBoundMethod; + void becomeInternalizedOf( + const ClosureVariableBinding& val, + PyObjSnapshotMaker& maker + ); - PyObjectStealer fself(PyObject_GetAttrString(val, "__self__")); - PyObjectStealer ffunc(PyObject_GetAttrString(val, "__func__")); + void becomeInternalizedOf( + const ClosureVariableBindingStep& val, + PyObjSnapshotMaker& maker + ); - mNamedElements["meth_self"] = internalize(fself); - mNamedElements["meth_func"] = internalize(ffunc); + void becomeInternalizedOf( + const MemberDefinition& val, + PyObjSnapshotMaker& maker + ); - return; - } + void becomeInternalizedOf( + const FunctionGlobal& val, + PyObjSnapshotMaker& maker + ); - if (isPyObjectGloballyIdentifiable(val)) { - mKind = Kind::NamedPyObject; + void becomeInternalizedOf( + const FunctionOverload& val, + PyObjSnapshotMaker& maker + ); - // no checks are necessary because isPyObjectGloballyIdentifiable - // confirms that this is OK. - PyObjectStealer moduleName(PyObject_GetAttrString(val, "__module__")); - PyObjectStealer clsName(PyObject_GetAttrString(val, "__name__")); + void becomeInternalizedOf( + InstanceRef val, + PyObjSnapshotMaker& maker + ); - mModuleName = PyUnicode_AsUTF8(moduleName); - mName = PyUnicode_AsUTF8(clsName); - return; - } + void becomeInternalizedOf( + Type* t, + PyObjSnapshotMaker& maker + ); - mKind = Kind::ArbitraryPyObject; - mPyObject = incref(val); - } + void becomeInternalizedOf( + PyObject* val, + PyObjSnapshotMaker& maker + ); void append(PyObjSnapshot* elt) { if (mKind != Kind::PyTuple) { @@ -1292,6 +1162,25 @@ class PyObjSnapshot { return result; } + template + void becomeBundleOf(const std::map& namedElements, PyObjSnapshotMaker& maker) { + mKind = Kind::InternalBundle; + + for (auto& nameAndElt: namedElements) { + mNames.push_back(nameAndElt.first); + mElements.push_back(maker.internalize(nameAndElt.second)); + } + } + + template + void becomeBundleOf(const std::vector& elements, PyObjSnapshotMaker& maker) { + mKind = Kind::InternalBundle; + + for (auto& elt: elements) { + mElements.push_back(maker.internalize(elt)); + } + } + private: // ensure we won't crash if we interact with this object. void validateAfterDeserialization() { @@ -1324,10 +1213,13 @@ class PyObjSnapshot { // if we're a tuple, list, or dict std::vector mElements; + std::vector mNames; + // if we're a PyDict std::vector mKeys; std::string mStringValue; std::string mModuleName; std::string mName; + std::string mQualname; }; diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index 55d21b93e..30d89745a 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -40,19 +40,25 @@ PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwar } std::unordered_map constantMapCache; + std::unordered_map constantMapCacheType; + std::unordered_map constantMapCacheInst; std::map<::Type*, ::Type*> groupMap; PyObjGraphSnapshot* graph = new PyObjGraphSnapshot(); - PyObjSnapshot* object = PyObjSnapshot::internalizePyObj( - instance, + PyObjSnapshotMaker maker( constantMapCache, + constantMapCacheType, + constantMapCacheInst, groupMap, - linkBack, - graph + graph, + linkBack ); + PyObjSnapshot* object = maker.internalize(instance); + PyObjectStealer graphObj(PyPyObjGraphSnapshot::newPyObjGraphSnapshot(graph, true)); + return PyPyObjSnapshot::newPyObjSnapshot(object, graphObj); }); } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 2a9d4fd9f..8865b25e9 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -477,12 +477,20 @@ void Type::attemptToResolve() { // allow each Function type to build CompilerVisiblePythonObjects for its globals // after this, any FunctionGlobals will refer to Constants not cells std::unordered_map compilerVisiblePyObjMap; + std::unordered_map compilerVisibleTypeMap; + std::unordered_map compilerVisibleInstanceMap; + PyObjSnapshotMaker snapMaker( + compilerVisiblePyObjMap, + compilerVisibleTypeMap, + compilerVisibleInstanceMap, + resolutionMapping, + nullptr, + true + ); + for (auto typeAndSource: resolutionSource) { if (typeAndSource.first->isFunction()) { - ((Function*)typeAndSource.first)->internalizeConstants( - compilerVisiblePyObjMap, - resolutionMapping - ); + ((Function*)typeAndSource.first)->internalizeConstants(snapMaker); } } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index ac9be33d3..2aa9a6dad 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -252,6 +252,10 @@ class Type { ); } + bool isEmbeddedMessage() const { + return m_typeCategory == catEmbeddedMessage; + } + bool isRegister() const { return ( m_typeCategory == catBool diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index 82e58254b..47dbc37ca 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -5,7 +5,7 @@ def test_make_snapshot_basic(): - assert PyObjSnapshot.create(int).kind == 'Type' + assert PyObjSnapshot.create(int).kind == 'PrimitiveType' assert PyObjSnapshot.create((1, 2, 3)).kind == 'PyTuple' assert PyObjSnapshot.create((1, 2, 3)).elements[0].kind == 'Instance' From 7b330b5ff34fdc670c04487f155b225af15c46de Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sat, 29 Jul 2023 22:34:12 +0000 Subject: [PATCH 66/83] Start to work out rehydrating Types. --- typed_python/PyObjRehydrator.cpp | 396 +++++++++++++++++ typed_python/PyObjRehydrator.hpp | 53 +++ typed_python/PyObjSnapshot.cpp | 226 +++++++++- typed_python/PyObjSnapshot.hpp | 729 ++++++------------------------- typed_python/PyPyObjSnapshot.cpp | 2 +- typed_python/Type.hpp | 4 + typed_python/all.cpp | 1 + 7 files changed, 799 insertions(+), 612 deletions(-) create mode 100644 typed_python/PyObjRehydrator.cpp create mode 100644 typed_python/PyObjRehydrator.hpp diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp new file mode 100644 index 000000000..bb1ae0dda --- /dev/null +++ b/typed_python/PyObjRehydrator.cpp @@ -0,0 +1,396 @@ +#include "PyObjRehydrator.hpp" +#include "PyObjSnapshot.hpp" + + +Type* PyObjRehydrator::typeFor(PyObjSnapshot* snapshot) { + if (snapshot->mType) { + return snapshot->mType; + } + + if (snapshot->willBeATpType()) { + rehydrate(snapshot); + return snapshot->mType; + } + + return nullptr; +} + + +PyObject* PyObjRehydrator::pyobjFor(PyObjSnapshot* snapshot) { + if (snapshot->mPyObject) { + return snapshot->mPyObject; + } + + rehydrate(snapshot); + + return snapshot->mPyObject; +} + + +PyObject* PyObjRehydrator::getNamedElementPyobj( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + if (allowEmpty) { + return nullptr; + } + throw std::runtime_error( + "Corrupt PyObjSnapshot." + snapshot->kindAsString() + ". missing " + name + ); + } + + return pyobjFor(it->second); +} + +Type* PyObjRehydrator::getNamedElementType( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + if (allowEmpty) { + return nullptr; + } + throw std::runtime_error( + "Corrupt PyObjSnapshot." + snapshot->kindAsString() + ". missing " + name + ); + } + + return typeFor(it->second); +} + +void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { + if (snap->mType) { + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ListOfType) { + snap->mType = new ListOfType(); + snap->mType->setIsForwardDefined(snap->mNamedInts["type_is_forward"]); + ((ListOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::TupleOfType) { + snap->mType = new TupleOfType(); + snap->mType->setIsForwardDefined(snap->mNamedInts["type_is_forward"]); + ((TupleOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); + return; + } + + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); +} + +void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { + if (mSnapshots.find(obj) != mSnapshots.end()) { + throw std::runtime_error("We should already have a rehydration for this object."); + } + + mSnapshots.insert(obj); + + if (obj->willBeATpType()) { + rehydrateTpType(obj); + return; + } + + static PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + + PyObjSnapshot::Kind kind = obj->mKind; + PyObject*& pyObject(obj->mPyObject); + + if (kind == PyObjSnapshot::Kind::ArbitraryPyObject) { + throw std::runtime_error("Corrupt PyObjSnapshot.ArbitraryPyObject: missing pyObject"); + } else if (kind == PyObjSnapshot::Kind::PrimitiveType) { + pyObject = (PyObject*)PyInstance::typeObj(obj->mType); + } else if (kind == PyObjSnapshot::Kind::String) { + pyObject = PyUnicode_FromString(obj->mStringValue.c_str()); + } else if (kind == PyObjSnapshot::Kind::Instance) { + pyObject = PyInstance::extractPythonObject(obj->mInstance); + } else if (kind == PyObjSnapshot::Kind::NamedPyObject) { + PyObjectStealer nameAsStr(PyUnicode_FromString(obj->mModuleName.c_str())); + PyObjectStealer moduleObject( + PyObject_GetItem(sysModuleModules, nameAsStr) + ); + + if (!moduleObject) { + PyErr_Clear(); + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + obj->mModuleName + " is not loaded!"); + } + PyObjectStealer inst(PyObject_GetAttrString(moduleObject, obj->mName.c_str())); + if (!inst) { + PyErr_Clear(); + + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + obj->mModuleName + " is missing member " + obj->mName); + } + + pyObject = incref(inst); + } else if (kind == PyObjSnapshot::Kind::PyDict || kind == PyObjSnapshot::Kind::PyClassDict) { + pyObject = PyDict_New(); + } else if (kind == PyObjSnapshot::Kind::PyList) { + pyObject = PyList_New(0); + + for (long k = 0; k < obj->mElements.size(); k++) { + PyList_Append(pyObject, pyobjFor(obj->mElements[k])); + } + } else if (kind == PyObjSnapshot::Kind::PyTuple) { + pyObject = PyTuple_New(obj->mElements.size()); + + // first initialize it in case we throw somehow + for (long k = 0; k < obj->mElements.size(); k++) { + PyTuple_SetItem(pyObject, k, incref(Py_None)); + } + + for (long k = 0; k < obj->mElements.size(); k++) { + PyTuple_SetItem(pyObject, k, incref(pyobjFor(obj->mElements[k]))); + } + } else if (kind == PyObjSnapshot::Kind::PySet) { + pyObject = PySet_New(nullptr); + + for (long k = 0; k < obj->mElements.size(); k++) { + PySet_Add(pyObject, incref(pyobjFor(obj->mElements[k]))); + } + } else if (kind == PyObjSnapshot::Kind::PyClass) { + PyObjectStealer argTup(PyTuple_New(3)); + PyTuple_SetItem(argTup, 0, PyUnicode_FromString(obj->mName.c_str())); + PyTuple_SetItem(argTup, 1, incref(getNamedElementPyobj(obj, "cls_bases"))); + PyTuple_SetItem(argTup, 2, incref(getNamedElementPyobj(obj, "cls_dict"))); + + pyObject = PyType_Type.tp_new(&PyType_Type, argTup, nullptr); + + if (!pyObject) { + throw PythonExceptionSet(); + } + } else if (kind == PyObjSnapshot::Kind::PyModule) { + PyObjectStealer nameAsStr(PyUnicode_FromString(obj->mName.c_str())); + PyObjectStealer moduleObject( + PyObject_GetItem(sysModuleModules, nameAsStr) + ); + + if (!moduleObject) { + PyErr_Clear(); + // TODO: should we be importing here? that seems dangerous... + throw std::runtime_error("Somehow module " + obj->mName + " is not loaded!"); + } + + pyObject = incref(moduleObject); + } else if (kind == PyObjSnapshot::Kind::PyModuleDict) { + pyObject = PyObject_GenericGetDict(getNamedElementPyobj(obj, "module_dict_of"), nullptr); + if (!pyObject) { + throw PythonExceptionSet(); + } + } else if (kind == PyObjSnapshot::Kind::PyCell) { + pyObject = PyCell_New(nullptr); + } else if (kind == PyObjSnapshot::Kind::PyObject) { + PyObject* t = getNamedElementPyobj(obj, "inst_type"); + PyObject* d = getNamedElementPyobj(obj, "inst_dict"); + + static PyObject* emptyTuple = PyTuple_Pack(0); + + pyObject = ((PyTypeObject*)t)->tp_new( + ((PyTypeObject*)t), + emptyTuple, + nullptr + ); + + if (PyObject_GenericSetDict(pyObject, d, nullptr)) { + throw PythonExceptionSet(); + } + } else if (kind == PyObjSnapshot::Kind::PyFunction) { + pyObject = PyFunction_New( + getNamedElementPyobj(obj, "func_code"), + getNamedElementPyobj(obj, "func_globals") + ); + + if (obj->mNamedElements.find("func_closure") != obj->mNamedElements.end()) { + PyFunction_SetClosure( + pyObject, + getNamedElementPyobj(obj, "func_closure") + ); + } + + if (obj->mNamedElements.find("func_annotations") != obj->mNamedElements.end()) { + PyFunction_SetAnnotations( + pyObject, + getNamedElementPyobj(obj, "func_annotations") + ); + } + + if (obj->mNamedElements.find("func_defaults") != obj->mNamedElements.end()) { + PyFunction_SetAnnotations( + pyObject, + getNamedElementPyobj(obj, "func_defaults") + ); + } + + if (obj->mNamedElements.find("func_qualname") != obj->mNamedElements.end()) { + PyObject_SetAttrString( + pyObject, + "__qualname__", + getNamedElementPyobj(obj, "func_qualname") + ); + } + + if (obj->mNamedElements.find("func_kwdefaults") != obj->mNamedElements.end()) { + PyObject_SetAttrString( + pyObject, + "__kwdefaults__", + getNamedElementPyobj(obj, "func_kwdefaults") + ); + } + + if (obj->mNamedElements.find("func_name") != obj->mNamedElements.end()) { + PyObject_SetAttrString( + pyObject, + "__name__", + getNamedElementPyobj(obj, "func_name") + ); + } + } else if (kind == PyObjSnapshot::Kind::PyCodeObject) { +#if PY_MINOR_VERSION < 8 + pyObject = (PyObject*)PyCode_New( +#else + pyObject = (PyObject*)PyCode_NewWithPosOnlyArgs( +#endif + obj->mNamedInts["co_argcount"], +#if PY_MINOR_VERSION >= 8 + obj->mNamedInts["co_posonlyargcount"], +#endif + obj->mNamedInts["co_kwonlyargcount"], + obj->mNamedInts["co_nlocals"], + obj->mNamedInts["co_stacksize"], + obj->mNamedInts["co_flags"], + getNamedElementPyobj(obj, "co_code"), + getNamedElementPyobj(obj, "co_consts"), + getNamedElementPyobj(obj, "co_names"), + getNamedElementPyobj(obj, "co_varnames"), + getNamedElementPyobj(obj, "co_freevars"), + getNamedElementPyobj(obj, "co_cellvars"), + getNamedElementPyobj(obj, "co_filename"), + getNamedElementPyobj(obj, "co_name"), + obj->mNamedInts["co_firstlineno"], + getNamedElementPyobj(obj, +#if PY_MINOR_VERSION >= 10 + "co_linetable" +#else + "co_lnotab" +#endif + ) + ); + } else if (kind == PyObjSnapshot::Kind::PyStaticMethod) { + pyObject = PyStaticMethod_New(Py_None); + + JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)pyObject; + decref(method->cm_callable); + method->cm_callable = incref(getNamedElementPyobj(obj, "meth_func")); + } else if (kind == PyObjSnapshot::Kind::PyClassMethod) { + pyObject = PyClassMethod_New(Py_None); + + JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)pyObject; + decref(method->cm_callable); + method->cm_callable = incref(getNamedElementPyobj(obj, "meth_func")); + } else if (kind == PyObjSnapshot::Kind::PyProperty) { + static PyObject* nones = PyTuple_Pack(3, Py_None, Py_None, Py_None); + + pyObject = PyObject_CallObject((PyObject*)&PyProperty_Type, nones); + + JustLikeAPropertyObject* dest = (JustLikeAPropertyObject*)pyObject; + + decref(dest->prop_get); + decref(dest->prop_set); + decref(dest->prop_del); + decref(dest->prop_doc); + + dest->prop_get = nullptr; + dest->prop_set = nullptr; + dest->prop_del = nullptr; + dest->prop_doc = nullptr; + + #if PY_MINOR_VERSION >= 10 + decref(dest->prop_name); + dest->prop_name = nullptr; + #endif + + dest->prop_get = incref(getNamedElementPyobj(obj, "prop_get", true)); + dest->prop_set = incref(getNamedElementPyobj(obj, "prop_set", true)); + dest->prop_del = incref(getNamedElementPyobj(obj, "prop_del", true)); + dest->prop_doc = incref(getNamedElementPyobj(obj, "prop_doc", true)); + + #if PY_MINOR_VERSION >= 10 + decref(dest->prop_name); + dest->prop_name = incref(getNamedElementPyobj(obj, "prop_name", true)); + #endif + } else if (kind == PyObjSnapshot::Kind::PyBoundMethod) { + pyObject = PyMethod_New(Py_None, Py_None); + PyMethodObject* method = (PyMethodObject*)pyObject; + decref(method->im_func); + decref(method->im_self); + method->im_func = nullptr; + method->im_self = nullptr; + + method->im_func = incref(getNamedElementPyobj(obj, "meth_func")); + method->im_self = incref(getNamedElementPyobj(obj, "meth_self")); + } else { + throw std::runtime_error( + "Can't make a python object representation for a PyObjSnapshot of kind " + + obj->kindAsString() + ); + } +} + + +void PyObjRehydrator::finalize() { + for (auto n: mSnapshots) { + finalizeRehydration(n); + } + + for (auto n: mSnapshots) { + finalizeRehydration2(n); + } +} + + +void PyObjRehydrator::finalizeRehydration2(PyObjSnapshot* obj) { + if (obj->mKind == PyObjSnapshot::Kind::PyClass) { + if (obj->mNamedElements.find("cls_dict") == obj->mNamedElements.end()) { + throw std::runtime_error("Corrupt PyClass - no cls_dict"); + } + + PyObjSnapshot* clsDictPyObj = obj->mNamedElements["cls_dict"]; + + for (long k = 0; k < clsDictPyObj->elements().size() && k < clsDictPyObj->keys().size(); k++) { + if (clsDictPyObj->keys()[k]->isString()) { + PyObject_SetAttrString( + obj->mPyObject, + clsDictPyObj->keys()[k]->getStringValue().c_str(), + clsDictPyObj->elements()[k]->getPyObj() + ); + } + } + } +} + +void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { + if (obj->mKind == PyObjSnapshot::Kind::PyDict || obj->mKind == PyObjSnapshot::Kind::PyClassDict) { + for (long k = 0; k < obj->mElements.size() && k < obj->mKeys.size(); k++) { + PyDict_SetItem( + obj->mPyObject, + obj->mKeys[k]->getPyObj(), + obj->mElements[k]->getPyObj() + ); + } + } else if (obj->mKind == PyObjSnapshot::Kind::PyCell) { + auto it = obj->mNamedElements.find("cell_contents"); + if (it != obj->mNamedElements.end()) { + PyCell_Set( + obj->mPyObject, + it->second->getPyObj() + ); + } + } +} diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp new file mode 100644 index 000000000..b2c6c693f --- /dev/null +++ b/typed_python/PyObjRehydrator.hpp @@ -0,0 +1,53 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include + +class PyObjSnapshot; + +class PyObjRehydrator { +public: + void rehydrate(PyObjSnapshot* snapshot); + + Type* typeFor(PyObjSnapshot* snapshot); + + PyObject* pyobjFor(PyObjSnapshot* snapshot); + + void finalize(); + + PyObject* getNamedElementPyobj( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty=false + ); + + Type* getNamedElementType( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty=false + ); + +private: + void rehydrateTpType(PyObjSnapshot* snap); + + void finalizeRehydration(PyObjSnapshot* snap); + + void finalizeRehydration2(PyObjSnapshot* snap); + + std::unordered_set mSnapshots; +}; diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index a0d637ad4..1c0beefbc 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -2,6 +2,203 @@ #include "PyObjGraphSnapshot.hpp" +std::string PyObjSnapshot::kindAsString() const { + if (mKind == Kind::Uninitialized) { + return "Uninitialized"; + } + + if (mKind == Kind::InternalBundle) { + return "InternalBundle"; + } + + if (mKind == Kind::String) { + return "String"; + } + + if (mKind == Kind::NamedPyObject) { + return "NamedPyObject"; + } + + if (mKind == Kind::PrimitiveType) { + return "PrimitiveType"; + } + + if (mKind == Kind::Instance) { + return "Instance"; + } + + if (mKind == Kind::PrimitiveType) { + return "PrimitiveType"; + } + + if (mKind == Kind::ListOfType) { + return "ListOfType"; + } + + if (mKind == Kind::TupleOfType) { + return "TupleOfType"; + } + + if (mKind == Kind::TupleType) { + return "TupleType"; + } + + if (mKind == Kind::NamedTupleType) { + return "NamedTupleType"; + } + + if (mKind == Kind::OneOfType) { + return "OneOfType"; + } + + if (mKind == Kind::ValueType) { + return "ValueType"; + } + + if (mKind == Kind::DictType) { + return "DictType"; + } + + if (mKind == Kind::ConstDictType) { + return "ConstDictType"; + } + + if (mKind == Kind::SetType) { + return "SetType"; + } + + if (mKind == Kind::PointerToType) { + return "PointerToType"; + } + + if (mKind == Kind::RefToType) { + return "RefToType"; + } + + if (mKind == Kind::AlternativeType) { + return "AlternativeType"; + } + + if (mKind == Kind::ConcreteAlternativeType) { + return "ConcreteAlternativeType"; + } + + if (mKind == Kind::AlternativeMatcherType) { + return "AlternativeMatcherType"; + } + + if (mKind == Kind::PythonObjectOfTypeType) { + return "PythonObjectOfTypeType"; + } + + if (mKind == Kind::SubclassOfTypeType) { + return "SubclassOfTypeType"; + } + + if (mKind == Kind::ClassType) { + return "ClassType"; + } + + if (mKind == Kind::HeldClassType) { + return "HeldClassType"; + } + + if (mKind == Kind::FunctionType) { + return "FunctionType"; + } + + if (mKind == Kind::FunctionOverload) { + return "FunctionOverload"; + } + + if (mKind == Kind::FunctionGlobal) { + return "FunctionGlobal"; + } + + if (mKind == Kind::BoundMethodType) { + return "BoundMethodType"; + } + + if (mKind == Kind::ForwardType) { + return "ForwardType"; + } + + if (mKind == Kind::TypedCellType) { + return "TypedCellType"; + } + + if (mKind == Kind::PyList) { + return "PyList"; + } + + if (mKind == Kind::PyDict) { + return "PyDict"; + } + + if (mKind == Kind::PySet) { + return "PySet"; + } + + if (mKind == Kind::PyTuple) { + return "PyTuple"; + } + + if (mKind == Kind::PyClass) { + return "PyClass"; + } + + if (mKind == Kind::PyFunction) { + return "PyFunction"; + } + + if (mKind == Kind::PyObject) { + return "PyObject"; + } + + if (mKind == Kind::PyCell) { + return "PyCell"; + } + + if (mKind == Kind::PyCodeObject) { + return "PyCodeObject"; + } + + if (mKind == Kind::PyModule) { + return "PyModule"; + } + + if (mKind == Kind::PyModuleDict) { + return "PyModuleDict"; + } + + if (mKind == Kind::PyClassDict) { + return "PyClassDict"; + } + + if (mKind == Kind::PyStaticMethod) { + return "PyStaticMethod"; + } + + if (mKind == Kind::PyClassMethod) { + return "PyClassMethod"; + } + + if (mKind == Kind::PyBoundMethod) { + return "PyBoundMethod"; + } + + if (mKind == Kind::PyProperty) { + return "PyProperty"; + } + + if (mKind == Kind::ArbitraryPyObject) { + return "ArbitraryPyObject"; + } + + throw std::runtime_error("Unknown PyObjSnapshot Kind: " + format((int)mKind)); +} + + PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); @@ -139,6 +336,17 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); if (mGraph) { @@ -262,6 +470,13 @@ void PyObjSnapshot::becomeInternalizedOf( Type* t, PyObjSnapshotMaker& maker ) { + // we're always the internalized version of this object + if (maker.linkBackToOriginalObject()) { + mType = t; + } + + mNamedInts["type_is_forward"] = t->isForwardDefined() ? 1 : 0; + if (t->isListOf()) { mKind = Kind::ListOfType; mNamedElements["element_type"] = maker.internalize(((ListOfType*)t)->getEltType()); @@ -475,8 +690,7 @@ void PyObjSnapshot::becomeInternalizedOf( return; } - mKind = Kind::Type; - mType = t; + throw std::runtime_error("Type of category " + t->getTypeCategoryString() + " can't be snapshotted"); } @@ -916,8 +1130,12 @@ void PyObjSnapshot::becomeInternalizedOf( mNamedElements["func_globals"] = maker.internalize(val.getGlobals()); mNamedElements["func_code"] = maker.internalize(val.getFunctionCode()); - mNamedElements["func_defaults"] = maker.internalize(val.getFunctionDefaults()); - mNamedElements["func_annotations"] = maker.internalize(val.getFunctionAnnotations()); + if (val.getFunctionDefaults()) { + mNamedElements["func_defaults"] = maker.internalize(val.getFunctionDefaults()); + } + if (val.getFunctionAnnotations()) { + mNamedElements["func_annotations"] = maker.internalize(val.getFunctionAnnotations()); + } mNamedElements["func_args"] = maker.internalize(val.getArgs()); mNamedElements["func_closure_varnames"] = maker.internalize( val.getFunctionClosureVarnames() diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index c0be98154..54083d070 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -20,6 +20,7 @@ #include "Type.hpp" #include "Instance.hpp" #include "PythonTypeInternals.hpp" +#include "PyObjRehydrator.hpp" class PyObjGraphSnapshot; class PyObjSnapshot; @@ -42,7 +43,6 @@ gives us a self-consistent view of the world to compile against so we don't have changing underneath us. **********************************/ - class PyObjSnapshotMaker { public: PyObjSnapshotMaker( @@ -109,6 +109,7 @@ class PyObjSnapshotMaker { class PyObjSnapshot { private: friend class PyObjSnapshotMaker; + friend class PyObjRehydrator; enum class Kind { // this should never be visible in a running program @@ -121,9 +122,6 @@ class PyObjSnapshot { // is the same across program invocations (its a C function and we can't // look inside of it) NamedPyObject, - // we're pointing back into a typed_python Type held in mType. - // it must be a 'leaf' type with no internals - Type, // we're pointing into a TP leaf instance (a register type, an int, bytes, etc.) Instance, // we're a primitive type (like int) @@ -238,8 +236,9 @@ class PyObjSnapshot { } static PyObjSnapshot* ForType(Type* t) { + // TODO: this needs to go away PyObjSnapshot* res = new PyObjSnapshot(); - res->mKind = Kind::Type; + res->mKind = Kind::PrimitiveType; res->mType = t; return res; } @@ -248,16 +247,8 @@ class PyObjSnapshot { return mGraph; } - bool isUninitialized() const { - return mKind == Kind::Uninitialized; - } - bool isType() const { - return mKind == Kind::Type; - } - - bool isNamedPyObject() const { - return mKind == Kind::NamedPyObject; + return mKind == Kind::PrimitiveType; } bool isString() const { @@ -268,245 +259,7 @@ class PyObjSnapshot { return mKind == Kind::Instance; } - bool isPyList() const { - return mKind == Kind::PyList; - } - - bool isPyDict() const { - return mKind == Kind::PyDict; - } - - bool isPySet() const { - return mKind == Kind::PySet; - } - - bool isPyTuple() const { - return mKind == Kind::PyTuple; - } - - bool isPyClass() const { - return mKind == Kind::PyClass; - } - - bool isPyFunction() const { - return mKind == Kind::PyFunction; - } - - bool isPyCell() const { - return mKind == Kind::PyCell; - } - - bool isPyObject() const { - return mKind == Kind::PyObject; - } - - bool isPyCodeObject() const { - return mKind == Kind::PyCodeObject; - } - - bool isPyModule() const { - return mKind == Kind::PyModule; - } - - bool isArbitraryPyObject() const { - return mKind == Kind::ArbitraryPyObject; - } - - std::string kindAsString() const { - if (mKind == Kind::Uninitialized) { - return "Uninitialized"; - } - - if (mKind == Kind::InternalBundle) { - return "InternalBundle"; - } - - if (mKind == Kind::String) { - return "String"; - } - - if (mKind == Kind::NamedPyObject) { - return "NamedPyObject"; - } - - if (mKind == Kind::Type) { - return "Type"; - } - - if (mKind == Kind::Instance) { - return "Instance"; - } - - if (mKind == Kind::PrimitiveType) { - return "PrimitiveType"; - } - - if (mKind == Kind::ListOfType) { - return "ListOfType"; - } - - if (mKind == Kind::TupleOfType) { - return "TupleOfType"; - } - - if (mKind == Kind::TupleType) { - return "TupleType"; - } - - if (mKind == Kind::NamedTupleType) { - return "NamedTupleType"; - } - - if (mKind == Kind::OneOfType) { - return "OneOfType"; - } - - if (mKind == Kind::ValueType) { - return "ValueType"; - } - - if (mKind == Kind::DictType) { - return "DictType"; - } - - if (mKind == Kind::ConstDictType) { - return "ConstDictType"; - } - - if (mKind == Kind::SetType) { - return "SetType"; - } - - if (mKind == Kind::PointerToType) { - return "PointerToType"; - } - - if (mKind == Kind::RefToType) { - return "RefToType"; - } - - if (mKind == Kind::AlternativeType) { - return "AlternativeType"; - } - - if (mKind == Kind::ConcreteAlternativeType) { - return "ConcreteAlternativeType"; - } - - if (mKind == Kind::AlternativeMatcherType) { - return "AlternativeMatcherType"; - } - - if (mKind == Kind::PythonObjectOfTypeType) { - return "PythonObjectOfTypeType"; - } - - if (mKind == Kind::SubclassOfTypeType) { - return "SubclassOfTypeType"; - } - - if (mKind == Kind::ClassType) { - return "ClassType"; - } - - if (mKind == Kind::HeldClassType) { - return "HeldClassType"; - } - - if (mKind == Kind::FunctionType) { - return "FunctionType"; - } - - if (mKind == Kind::FunctionOverload) { - return "FunctionOverload"; - } - - if (mKind == Kind::FunctionGlobal) { - return "FunctionGlobal"; - } - - if (mKind == Kind::BoundMethodType) { - return "BoundMethodType"; - } - - if (mKind == Kind::ForwardType) { - return "ForwardType"; - } - - if (mKind == Kind::TypedCellType) { - return "TypedCellType"; - } - - if (mKind == Kind::PyList) { - return "PyList"; - } - - if (mKind == Kind::PyDict) { - return "PyDict"; - } - - if (mKind == Kind::PySet) { - return "PySet"; - } - - if (mKind == Kind::PyTuple) { - return "PyTuple"; - } - - if (mKind == Kind::PyClass) { - return "PyClass"; - } - - if (mKind == Kind::PyFunction) { - return "PyFunction"; - } - - if (mKind == Kind::PyObject) { - return "PyObject"; - } - - if (mKind == Kind::PyCell) { - return "PyCell"; - } - - if (mKind == Kind::PyCodeObject) { - return "PyCodeObject"; - } - - if (mKind == Kind::PyModule) { - return "PyModule"; - } - - if (mKind == Kind::PyModuleDict) { - return "PyModuleDict"; - } - - if (mKind == Kind::PyClassDict) { - return "PyClassDict"; - } - - if (mKind == Kind::PyStaticMethod) { - return "PyStaticMethod"; - } - - if (mKind == Kind::PyClassMethod) { - return "PyClassMethod"; - } - - if (mKind == Kind::PyBoundMethod) { - return "PyBoundMethod"; - } - - if (mKind == Kind::PyProperty) { - return "PyProperty"; - } - - if (mKind == Kind::ArbitraryPyObject) { - return "ArbitraryPyObject"; - } - - throw std::runtime_error("Unknown PyObjSnapshot Kind: " + format((int)mKind)); - } + std::string kindAsString() const; static std::string dictGetStringOrEmpty(PyObject* dict, const char* name) { if (!dict || !PyDict_Check(dict)) { @@ -662,14 +415,6 @@ class PyObjSnapshot { return mNamedInts; } - ::Type* getType() const { - return mType; - } - - const ::Instance& getInstance() const { - return mInstance; - } - const std::string& getStringValue() const { return mStringValue; } @@ -684,7 +429,7 @@ class PyObjSnapshot { template void _visitReferencedTypes(const visitor_type& v) { - if (mKind == Kind::Type) { + if (mKind == Kind::PrimitiveType) { v(mType); return; } @@ -700,7 +445,7 @@ class PyObjSnapshot { template void _visitCompilerVisibleInternals(const visitor_type& visitor) { - if (mKind == Kind::Type) { + if (mKind == Kind::PrimitiveType) { visitor.visitTopo(mType); } @@ -719,347 +464,127 @@ class PyObjSnapshot { } } - // get the python object representation of this object, which isn't guaranteed - // to exist and may need to be constructed on demand. this will do a pass over - // all reachable objects, building skeletons, and then performs a second pass where - // we fill items out once we have the skeletons in place. - PyObject* getPyObj() { - if (mPyObject) { - return mPyObject; - } - - std::unordered_set needsResolution; - - PyObject* res = getPyObj(needsResolution); - - for (auto n: needsResolution) { - n->finalizeGetPyObj(); + bool willBeATpType() const { + return mKind == Kind::PrimitiveType + || mKind == Kind::ListOfType + || mKind == Kind::TupleOfType + || mKind == Kind::TupleType + || mKind == Kind::NamedTupleType + || mKind == Kind::OneOfType + || mKind == Kind::ValueType + || mKind == Kind::DictType + || mKind == Kind::ConstDictType + || mKind == Kind::SetType + || mKind == Kind::PointerToType + || mKind == Kind::RefToType + || mKind == Kind::AlternativeType + || mKind == Kind::ConcreteAlternativeType + || mKind == Kind::AlternativeMatcherType + || mKind == Kind::PythonObjectOfTypeType + || mKind == Kind::SubclassOfTypeType + || mKind == Kind::ClassType + || mKind == Kind::HeldClassType + || mKind == Kind::FunctionType + || mKind == Kind::BoundMethodType + || mKind == Kind::ForwardType + || mKind == Kind::TypedCellType + ; + } + + + ::Type* getType() { + if (mKind == Kind::PrimitiveType) { + return mType; } - // do a second pass patching up classes. They don't inherit things - // from their dict correctly - for (auto n: needsResolution) { - n->finalizeGetPyObj2(); + if (willBeATpType() && !mType) { + rehydrate(); + return mType; } - return res; + return nullptr; } - void finalizeGetPyObj2() { - if (mKind == Kind::PyClass) { - if (mNamedElements.find("cls_dict") == mNamedElements.end()) { - throw std::runtime_error("Corrupt PyClass - no cls_dict"); - } - - PyObjSnapshot* clsDictPyObj = mNamedElements["cls_dict"]; - - for (long k = 0; k < clsDictPyObj->elements().size() && k < clsDictPyObj->keys().size(); k++) { - if (clsDictPyObj->keys()[k]->isString()) { - PyObject_SetAttrString( - mPyObject, - clsDictPyObj->keys()[k]->getStringValue().c_str(), - clsDictPyObj->elements()[k]->getPyObj() - ); - } - } - } + const ::Instance& getInstance() { + return mInstance; } - // we're a skeleton - finish ourselves out - void finalizeGetPyObj() { - if (mKind == Kind::PyDict || mKind == Kind::PyClassDict) { - for (long k = 0; k < mElements.size() && k < mKeys.size(); k++) { - PyDict_SetItem( - mPyObject, - mKeys[k]->getPyObj(), - mElements[k]->getPyObj() - ); - } - } else if (mKind == Kind::PyCell) { - auto it = mNamedElements.find("cell_contents"); - if (it != mNamedElements.end()) { - PyCell_Set( - mPyObject, - it->second->getPyObj() - ); - } + bool needsHydration() const { + // currently, instances are 'out of band'. Eventually they'll be just like the + // other content. + if (mKind == Kind::Instance) { + return false; } - } - - PyObject* getPyObj(std::unordered_set& needsResolution) { - PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); - if (!mPyObject) { - if (mKind == Kind::ArbitraryPyObject) { - throw std::runtime_error("Corrupt PyObjSnapshot.ArbitraryPyObject: missing mPyObject"); - } else if (mKind == Kind::Type) { - mPyObject = (PyObject*)PyInstance::typeObj(mType); - } else if (mKind == Kind::String) { - mPyObject = PyUnicode_FromString(mStringValue.c_str()); - } else if (mKind == Kind::Instance) { - mPyObject = PyInstance::extractPythonObject(mInstance); - } else if (mKind == Kind::NamedPyObject) { - PyObjectStealer nameAsStr(PyUnicode_FromString(mModuleName.c_str())); - PyObjectStealer moduleObject( - PyObject_GetItem(sysModuleModules, nameAsStr) - ); - - if (!moduleObject) { - PyErr_Clear(); - // TODO: should we be importing here? that seems dangerous... - throw std::runtime_error("Somehow module " + mModuleName + " is not loaded!"); - } - PyObjectStealer inst(PyObject_GetAttrString(moduleObject, mName.c_str())); - if (!inst) { - PyErr_Clear(); - - // TODO: should we be importing here? that seems dangerous... - throw std::runtime_error("Somehow module " + mModuleName + " is missing member " + mName); - } - - mPyObject = incref(inst); - } else if (mKind == Kind::PyDict || mKind == Kind::PyClassDict) { - mPyObject = PyDict_New(); - needsResolution.insert(this); - } else if (mKind == Kind::PyList) { - mPyObject = PyList_New(0); - - for (long k = 0; k < mElements.size(); k++) { - PyList_Append(mPyObject, mElements[k]->getPyObj(needsResolution)); - } - } else if (mKind == Kind::PyTuple) { - mPyObject = PyTuple_New(mElements.size()); - - // first initialize it in case we throw somehow - for (long k = 0; k < mElements.size(); k++) { - PyTuple_SetItem(mPyObject, k, incref(Py_None)); - } - - for (long k = 0; k < mElements.size(); k++) { - PyTuple_SetItem(mPyObject, k, incref(mElements[k]->getPyObj(needsResolution))); - } - } else if (mKind == Kind::PySet) { - mPyObject = PySet_New(nullptr); - - for (long k = 0; k < mElements.size(); k++) { - PySet_Add(mPyObject, incref(mElements[k]->getPyObj(needsResolution))); - } - } else if (mKind == Kind::PyClass) { - needsResolution.insert(this); - - PyObjectStealer argTup(PyTuple_New(3)); - PyTuple_SetItem(argTup, 0, PyUnicode_FromString(mName.c_str())); - PyTuple_SetItem(argTup, 1, incref(getNamedElementPyobj("cls_bases", needsResolution))); - PyTuple_SetItem(argTup, 2, incref(getNamedElementPyobj("cls_dict", needsResolution))); - - mPyObject = PyType_Type.tp_new(&PyType_Type, argTup, nullptr); - - if (!mPyObject) { - throw PythonExceptionSet(); - } - } else if (mKind == Kind::PyModule) { - PyObjectStealer nameAsStr(PyUnicode_FromString(mName.c_str())); - PyObjectStealer moduleObject( - PyObject_GetItem(sysModuleModules, nameAsStr) - ); - - if (!moduleObject) { - PyErr_Clear(); - // TODO: should we be importing here? that seems dangerous... - throw std::runtime_error("Somehow module " + mName + " is not loaded!"); - } + if (mKind == Kind::String) { + return false; + } - mPyObject = incref(moduleObject); - } else if (mKind == Kind::PyModuleDict) { - mPyObject = PyObject_GenericGetDict(getNamedElementPyobj("module_dict_of", needsResolution), nullptr); - if (!mPyObject) { - throw PythonExceptionSet(); - } - } else if (mKind == Kind::PyCell) { - mPyObject = PyCell_New(nullptr); - needsResolution.insert(this); - } else if (mKind == Kind::PyObject) { - PyObject* t = getNamedElementPyobj("inst_type", needsResolution); - PyObject* d = getNamedElementPyobj("inst_dict", needsResolution); - - static PyObject* emptyTuple = PyTuple_Pack(0); - - mPyObject = ((PyTypeObject*)t)->tp_new( - ((PyTypeObject*)t), - emptyTuple, - nullptr - ); - - if (PyObject_GenericSetDict(mPyObject, d, nullptr)) { - throw PythonExceptionSet(); - } - } else if (mKind == Kind::PyFunction) { - mPyObject = PyFunction_New( - getNamedElementPyobj("func_code", needsResolution), - getNamedElementPyobj("func_globals", needsResolution) - ); - - if (mNamedElements.find("func_closure") != mNamedElements.end()) { - PyFunction_SetClosure( - mPyObject, - getNamedElementPyobj("func_closure", needsResolution) - ); - } + if (mKind == Kind::Uninitialized || mKind == Kind::InternalBundle) { + return false; + } - if (mNamedElements.find("func_annotations") != mNamedElements.end()) { - PyFunction_SetAnnotations( - mPyObject, - getNamedElementPyobj("func_annotations", needsResolution) - ); - } + if (mPyObject) { + return false; + } - if (mNamedElements.find("func_defaults") != mNamedElements.end()) { - PyFunction_SetAnnotations( - mPyObject, - getNamedElementPyobj("func_defaults", needsResolution) - ); - } + return true; + } - if (mNamedElements.find("func_qualname") != mNamedElements.end()) { - PyObject_SetAttrString( - mPyObject, - "__qualname__", - getNamedElementPyobj("func_qualname", needsResolution) - ); - } + // get the python object representation of this object, which isn't guaranteed + // to exist and may need to be constructed on demand. this will do a pass over + // all reachable objects, building skeletons, and then performs a second pass where + // we fill items out once we have the skeletons in place. + PyObject* getPyObj() { + if (mPyObject) { + return mPyObject; + } - if (mNamedElements.find("func_kwdefaults") != mNamedElements.end()) { - PyObject_SetAttrString( - mPyObject, - "__kwdefaults__", - getNamedElementPyobj("func_kwdefaults", needsResolution) - ); - } + if (mKind == Kind::String) { + mPyObject = PyUnicode_FromString(mStringValue.c_str()); + return mPyObject; + } - if (mNamedElements.find("func_name") != mNamedElements.end()) { - PyObject_SetAttrString( - mPyObject, - "__name__", - getNamedElementPyobj("func_name", needsResolution) - ); - } - } else if (mKind == Kind::PyCodeObject) { -#if PY_MINOR_VERSION < 8 - mPyObject = (PyObject*)PyCode_New( -#else - mPyObject = (PyObject*)PyCode_NewWithPosOnlyArgs( -#endif - mNamedInts["co_argcount"], -#if PY_MINOR_VERSION >= 8 - mNamedInts["co_posonlyargcount"], -#endif - mNamedInts["co_kwonlyargcount"], - mNamedInts["co_nlocals"], - mNamedInts["co_stacksize"], - mNamedInts["co_flags"], - getNamedElementPyobj("co_code", needsResolution), - getNamedElementPyobj("co_consts", needsResolution), - getNamedElementPyobj("co_names", needsResolution), - getNamedElementPyobj("co_varnames", needsResolution), - getNamedElementPyobj("co_freevars", needsResolution), - getNamedElementPyobj("co_cellvars", needsResolution), - getNamedElementPyobj("co_filename", needsResolution), - getNamedElementPyobj("co_name", needsResolution), - mNamedInts["co_firstlineno"], - getNamedElementPyobj( -#if PY_MINOR_VERSION >= 10 - "co_linetable", -#else - "co_lnotab", -#endif - needsResolution - ) - ); - } else if (mKind == Kind::PyStaticMethod) { - mPyObject = PyStaticMethod_New(Py_None); - - JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)mPyObject; - decref(method->cm_callable); - method->cm_callable = incref(getNamedElementPyobj("meth_func", needsResolution)); - } else if (mKind == Kind::PyClassMethod) { - mPyObject = PyClassMethod_New(Py_None); - - JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)mPyObject; - decref(method->cm_callable); - method->cm_callable = incref(getNamedElementPyobj("meth_func", needsResolution)); - } else if (mKind == Kind::PyProperty) { - static PyObject* nones = PyTuple_Pack(3, Py_None, Py_None, Py_None); - - mPyObject = PyObject_CallObject((PyObject*)&PyProperty_Type, nones); - - JustLikeAPropertyObject* dest = (JustLikeAPropertyObject*)mPyObject; - - decref(dest->prop_get); - decref(dest->prop_set); - decref(dest->prop_del); - decref(dest->prop_doc); - - dest->prop_get = nullptr; - dest->prop_set = nullptr; - dest->prop_del = nullptr; - dest->prop_doc = nullptr; - - #if PY_MINOR_VERSION >= 10 - decref(dest->prop_name); - dest->prop_name = nullptr; - #endif - - dest->prop_get = incref(getNamedElementPyobj("prop_get", needsResolution, true)); - dest->prop_set = incref(getNamedElementPyobj("prop_set", needsResolution, true)); - dest->prop_del = incref(getNamedElementPyobj("prop_del", needsResolution, true)); - dest->prop_doc = incref(getNamedElementPyobj("prop_doc", needsResolution, true)); - - #if PY_MINOR_VERSION >= 10 - decref(dest->prop_name); - dest->prop_name = incref(getNamedElementPyobj("prop_name", needsResolution, true)); - #endif - } else if (mKind == Kind::PyBoundMethod) { - mPyObject = PyMethod_New(Py_None, Py_None); - PyMethodObject* method = (PyMethodObject*)mPyObject; - decref(method->im_func); - decref(method->im_self); - method->im_func = nullptr; - method->im_self = nullptr; - - method->im_func = incref(getNamedElementPyobj("meth_func", needsResolution)); - method->im_self = incref(getNamedElementPyobj("meth_self", needsResolution)); - } else { - throw std::runtime_error( - "Can't make a python object representation for a CVPO of kind " + kindAsString() - ); - } + if (mKind == Kind::Instance) { + mPyObject = PyInstance::extractPythonObject(mInstance); + return mPyObject; } + rehydrate(); return mPyObject; } - PyObject* getNamedElementPyobj( - std::string name, - std::unordered_set& needsResolution, - bool allowEmpty=false - ) { - auto it = mNamedElements.find(name); - if (it == mNamedElements.end()) { - if (allowEmpty) { - return nullptr; - } - throw std::runtime_error( - "Corrupt PyObjSnapshot." + kindAsString() + ". missing " + name - ); - } + void rehydrate() { + PyObjRehydrator rehydrator; - return it->second->getPyObj(needsResolution); + rehydrator.rehydrate(this); + rehydrator.finalize(); } std::string toString() const { return "PyObjSnapshot." + kindAsString() + "()"; } + template + void becomeBundleOf(const std::map& namedElements, PyObjSnapshotMaker& maker) { + mKind = Kind::InternalBundle; + + for (auto& nameAndElt: namedElements) { + mNames.push_back(nameAndElt.first); + mElements.push_back(maker.internalize(nameAndElt.second)); + } + } + + template + void becomeBundleOf(const std::vector& elements, PyObjSnapshotMaker& maker) { + mKind = Kind::InternalBundle; + + for (auto& elt: elements) { + mElements.push_back(maker.internalize(elt)); + } + } + + // TODO: fix serialization. This is just not right. template void serialize(serialization_context_t& context, buf_t& buffer, int fieldNumber) const { uint32_t id; @@ -1076,7 +601,8 @@ class PyObjSnapshot { buffer.writeUnsignedVarintObject(0, id); buffer.writeUnsignedVarintObject(1, (int)mKind); - if (mKind == Kind::Type) { + // we only serialize Type, Instance, and PyObj if they are not caches. + if (mKind == Kind::PrimitiveType) { context.serializeNativeType(mType, buffer, 2); } else if (mKind == Kind::Instance) { @@ -1085,15 +611,15 @@ class PyObjSnapshot { mInstance.type()->serialize(mInstance.data(), buffer, 1); buffer.writeEndCompound(); } else - if (mKind == Kind::PyTuple) { + if (mKind == Kind::ArbitraryPyObject) { + context.serializePythonObject(mPyObject, buffer, 5); + } else + if (mElements.size()) { buffer.writeBeginCompound(4); for (long i = 0; i < mElements.size(); i++) { mElements[i]->serialize(context, buffer, i); } buffer.writeEndCompound(); - } else - if (mKind == Kind::ArbitraryPyObject) { - context.serializePythonObject(mPyObject, buffer, 5); } buffer.writeEndCompound(); @@ -1162,37 +688,18 @@ class PyObjSnapshot { return result; } - template - void becomeBundleOf(const std::map& namedElements, PyObjSnapshotMaker& maker) { - mKind = Kind::InternalBundle; - - for (auto& nameAndElt: namedElements) { - mNames.push_back(nameAndElt.first); - mElements.push_back(maker.internalize(nameAndElt.second)); - } - } - - template - void becomeBundleOf(const std::vector& elements, PyObjSnapshotMaker& maker) { - mKind = Kind::InternalBundle; - - for (auto& elt: elements) { - mElements.push_back(maker.internalize(elt)); - } - } - private: // ensure we won't crash if we interact with this object. void validateAfterDeserialization() { - if (mKind == Kind::Type) { + if (mKind == Kind::PrimitiveType) { if (!mType) { - throw std::runtime_error("Corrupt CVPO: no Type"); + throw std::runtime_error("Corrupt PyObjSnapshot: no Type"); } } if (mKind == Kind::ArbitraryPyObject) { if (!mPyObject) { - throw std::runtime_error("Corrupt CVPO: no PyObject"); + throw std::runtime_error("Corrupt PyObjSnapshot: no PyObject"); } } } @@ -1200,26 +707,34 @@ class PyObjSnapshot { Kind mKind; PyObjGraphSnapshot* mGraph; + // if we are a PrimitiveType, then our type. If we resolve to a Type, a representation + // of us as that type. Otherwise nullptr. ::Type* mType; + + // if we are a PrimitiveInstance, the instance value. Otherwise a cache of our value + // as an instance. ::Instance mInstance; - // if we are an ArbitraryPythonObject this is always populated - // otherwise, it will be a cache for a constructed instance of the object + // if we are an ArbitraryPythonObject, then our value. Otherwise, a cache of our value + // as a PyObject. PyObject* mPyObject; + // these are the actual data that represent us std::map mNamedElements; + std::map mNamedInts; - // if we're a tuple, list, or dict std::vector mElements; std::vector mNames; - // if we're a PyDict std::vector mKeys; std::string mStringValue; + std::string mModuleName; + std::string mName; + std::string mQualname; }; diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index 30d89745a..b0db80e35 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -131,7 +131,7 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { if (attr == "pyobj") { PyObject* o = obj->getPyObj(); if (!o) { - throw std::runtime_error("CVPO of kind " + obj->kindAsString() + " has no pyobj"); + throw std::runtime_error("PyObjSnapshot of kind " + obj->kindAsString() + " has no pyobj"); } return incref(o); } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 2aa9a6dad..56fa1041a 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -920,6 +920,10 @@ class Type { void attemptAutoresolveWrite(); + void setIsForwardDefined(bool isForwardDefined) { + m_is_forward_defined = isForwardDefined; + } + void markActivelyBeingDeserialized(bool isForwardDefined) { m_is_being_deserialized = true; m_is_forward_defined = isForwardDefined; diff --git a/typed_python/all.cpp b/typed_python/all.cpp index 7538e7c99..b302539dd 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -79,6 +79,7 @@ compile the entire group all at once. #include "PyFunctionOverload.cpp" #include "PyFunctionGlobal.cpp" #include "PyObjSnapshot.cpp" +#include "PyObjRehydrator.cpp" #include "PyPyObjSnapshot.cpp" #include "PyPyObjGraphSnapshot.cpp" From cecf97d73a6cf0a26d8fc5c5538ee82413b04f58 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sun, 30 Jul 2023 00:38:40 +0000 Subject: [PATCH 67/83] Refactor how we do rehydration. --- typed_python/OneOfType.hpp | 4 + typed_python/PyObjRehydrator.cpp | 195 +++++++++++++++--- typed_python/PyObjRehydrator.hpp | 6 +- typed_python/PyObjSnapshot.hpp | 10 +- ...onSerializationContext_deserialization.cpp | 16 ++ typed_python/ValueType.hpp | 4 + typed_python/py_obj_snapshot_test.py | 5 +- 7 files changed, 208 insertions(+), 32 deletions(-) diff --git a/typed_python/OneOfType.hpp b/typed_python/OneOfType.hpp index f4dc06b51..a9fa712b4 100644 --- a/typed_python/OneOfType.hpp +++ b/typed_python/OneOfType.hpp @@ -149,6 +149,10 @@ class OneOfType : public Type { static OneOfType* Make(const std::vector& types); + void initializeDuringDeserialization(const std::vector& types) { + m_types = types; + } + Type* cloneForForwardResolutionConcrete(); void initializeFromConcrete(Type* forwardDefinitionOfSelf); diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index bb1ae0dda..8d08163ff 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -1,10 +1,41 @@ #include "PyObjRehydrator.hpp" #include "PyObjSnapshot.hpp" +void PyObjRehydrator::start(PyObjSnapshot* snapshot) { + std::function visit = [&](PyObjSnapshot* snap) { + if (mSnapshots.find(snap) != mSnapshots.end()) { + return; + } + + if (!snap->needsHydration()) { + return; + } + + mSnapshots.insert(snap); + + for (auto elt: snap->mElements) { + visit(elt); + } + + for (auto elt: snap->mKeys) { + visit(elt); + } + + for (auto& nameAndElt: snap->mNamedElements) { + visit(nameAndElt.second); + } + }; + + visit(snapshot); + + for (auto s: mSnapshots) { + rehydrate(s); + } +} Type* PyObjRehydrator::typeFor(PyObjSnapshot* snapshot) { - if (snapshot->mType) { - return snapshot->mType; + if (!snapshot->needsHydration()) { + return snapshot->getType(); } if (snapshot->willBeATpType()) { @@ -17,8 +48,8 @@ Type* PyObjRehydrator::typeFor(PyObjSnapshot* snapshot) { PyObject* PyObjRehydrator::pyobjFor(PyObjSnapshot* snapshot) { - if (snapshot->mPyObject) { - return snapshot->mPyObject; + if (!snapshot->needsHydration()) { + return snapshot->getPyObj(); } rehydrate(snapshot); @@ -70,38 +101,149 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { if (snap->mKind == PyObjSnapshot::Kind::ListOfType) { snap->mType = new ListOfType(); - snap->mType->setIsForwardDefined(snap->mNamedInts["type_is_forward"]); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((ListOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); return; } if (snap->mKind == PyObjSnapshot::Kind::TupleOfType) { snap->mType = new TupleOfType(); - snap->mType->setIsForwardDefined(snap->mNamedInts["type_is_forward"]); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((TupleOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); return; } + if (snap->mKind == PyObjSnapshot::Kind::DictType) { + snap->mType = new DictType(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + ((DictType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "key_type"), + getNamedElementType(snap, "value_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ConstDictType) { + snap->mType = new ConstDictType(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + ((ConstDictType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "key_type"), + getNamedElementType(snap, "value_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::SetType) { + snap->mType = new SetType(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + ((SetType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "key_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::PointerToType) { + snap->mType = new PointerTo(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + ((PointerTo*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "element_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::RefToType) { + snap->mType = new RefTo(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + ((RefTo*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "element_type") + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ValueType) { + snap->mType = new Value(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + if (snap->mNamedElements.find("value_instance") == snap->mNamedElements.end()) { + throw std::runtime_error("Corrupt PyObjSnapshot::Value"); + } + + rehydrate(snap->mNamedElements["value_instance"]); + + ((Value*)snap->mType)->initializeDuringDeserialization( + snap->mNamedElements["value_instance"]->mInstance + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::OneOfType) { + snap->mType = new OneOfType(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + std::vector types; + for (auto s: snap->mElements) { + types.push_back(typeFor(s)); + } + + ((OneOfType*)snap->mType)->initializeDuringDeserialization(types); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::TupleType) { + snap->mType = new Tuple(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + std::vector types; + for (auto s: snap->mElements) { + types.push_back(typeFor(s)); + } + + ((Tuple*)snap->mType)->initializeDuringDeserialization(types); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::NamedTupleType) { + snap->mType = new NamedTuple(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + std::vector types; + for (auto s: snap->mElements) { + types.push_back(typeFor(s)); + } + + ((NamedTuple*)snap->mType)->initializeDuringDeserialization(types, snap->mNames); + + return; + } + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); } void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { - if (mSnapshots.find(obj) != mSnapshots.end()) { - throw std::runtime_error("We should already have a rehydration for this object."); + if (mSnapshots.find(obj) == mSnapshots.end()) { + throw std::runtime_error( + "PyObjRehydrator walk didn't pick up snap of type " + + obj->kindAsString() + ); } - mSnapshots.insert(obj); - if (obj->willBeATpType()) { rehydrateTpType(obj); return; } - static PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); - PyObjSnapshot::Kind kind = obj->mKind; PyObject*& pyObject(obj->mPyObject); + // already rehydrated + if (pyObject) { + return; + } + + static PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + if (kind == PyObjSnapshot::Kind::ArbitraryPyObject) { throw std::runtime_error("Corrupt PyObjSnapshot.ArbitraryPyObject: missing pyObject"); } else if (kind == PyObjSnapshot::Kind::PrimitiveType) { @@ -343,19 +485,13 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { } } - void PyObjRehydrator::finalize() { for (auto n: mSnapshots) { finalizeRehydration(n); } - - for (auto n: mSnapshots) { - finalizeRehydration2(n); - } } - -void PyObjRehydrator::finalizeRehydration2(PyObjSnapshot* obj) { +void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { if (obj->mKind == PyObjSnapshot::Kind::PyClass) { if (obj->mNamedElements.find("cls_dict") == obj->mNamedElements.end()) { throw std::runtime_error("Corrupt PyClass - no cls_dict"); @@ -363,7 +499,8 @@ void PyObjRehydrator::finalizeRehydration2(PyObjSnapshot* obj) { PyObjSnapshot* clsDictPyObj = obj->mNamedElements["cls_dict"]; - for (long k = 0; k < clsDictPyObj->elements().size() && k < clsDictPyObj->keys().size(); k++) { + for (long k = 0; k < clsDictPyObj->elements().size() + && k < clsDictPyObj->keys().size(); k++) { if (clsDictPyObj->keys()[k]->isString()) { PyObject_SetAttrString( obj->mPyObject, @@ -372,19 +509,18 @@ void PyObjRehydrator::finalizeRehydration2(PyObjSnapshot* obj) { ); } } - } -} - -void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { + } else if (obj->mKind == PyObjSnapshot::Kind::PyDict || obj->mKind == PyObjSnapshot::Kind::PyClassDict) { - for (long k = 0; k < obj->mElements.size() && k < obj->mKeys.size(); k++) { + for (long k = 0; k < obj->mElements.size() + && k < obj->mKeys.size(); k++) { PyDict_SetItem( obj->mPyObject, obj->mKeys[k]->getPyObj(), obj->mElements[k]->getPyObj() ); } - } else if (obj->mKind == PyObjSnapshot::Kind::PyCell) { + } else + if (obj->mKind == PyObjSnapshot::Kind::PyCell) { auto it = obj->mNamedElements.find("cell_contents"); if (it != obj->mNamedElements.end()) { PyCell_Set( @@ -392,5 +528,12 @@ void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { it->second->getPyObj() ); } + } else + if (obj->mType) { + obj->mType->recomputeName(); + obj->mType->finalizeType(); + obj->mType->typeFinishedBeingDeserializedPhase1(); + obj->mType->typeFinishedBeingDeserializedPhase2(); + obj->mPyObject = incref((PyObject*)PyInstance::typeObj(obj->mType)); } } diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp index b2c6c693f..c3e812ad0 100644 --- a/typed_python/PyObjRehydrator.hpp +++ b/typed_python/PyObjRehydrator.hpp @@ -22,7 +22,7 @@ class PyObjSnapshot; class PyObjRehydrator { public: - void rehydrate(PyObjSnapshot* snapshot); + void start(PyObjSnapshot* snapshot); Type* typeFor(PyObjSnapshot* snapshot); @@ -43,11 +43,11 @@ class PyObjRehydrator { ); private: + void rehydrate(PyObjSnapshot* snapshot); + void rehydrateTpType(PyObjSnapshot* snap); void finalizeRehydration(PyObjSnapshot* snap); - void finalizeRehydration2(PyObjSnapshot* snap); - std::unordered_set mSnapshots; }; diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 54083d070..0a3bfa291 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -236,7 +236,8 @@ class PyObjSnapshot { } static PyObjSnapshot* ForType(Type* t) { - // TODO: this needs to go away + // TODO: this needs to go away. its a shim to handle the current + // code which is a little confused about these objects. PyObjSnapshot* res = new PyObjSnapshot(); res->mKind = Kind::PrimitiveType; res->mType = t; @@ -540,6 +541,11 @@ class PyObjSnapshot { return mPyObject; } + if (mType) { + mPyObject = incref((PyObject*)PyInstance::typeObj(mType)); + return mPyObject; + } + if (mKind == Kind::String) { mPyObject = PyUnicode_FromString(mStringValue.c_str()); return mPyObject; @@ -557,7 +563,7 @@ class PyObjSnapshot { void rehydrate() { PyObjRehydrator rehydrator; - rehydrator.rehydrate(this); + rehydrator.start(this); rehydrator.finalize(); } diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index f6be5ef68..30043ee6a 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -1998,6 +1998,22 @@ void PythonSerializationContext::deserializeNativeTypeIntoBlank( (PyTypeObject*)(PyObject*)obj ); } + else if (category == Type::TypeCategory::catValue) { + if (!blankShell->isValue()) { + throw std::runtime_error("Shell is not a Value"); + } + + ((Value*)blankShell)->initializeDuringDeserialization(instance); + } + else if (category == Type::TypeCategory::catOneOf) { + if (!blankShell->isOneOf()) { + throw std::runtime_error("Shell is not a OneOf"); + } + + ((OneOfType*)blankShell)->initializeDuringDeserialization( + types + ); + } else if (category == Type::TypeCategory::catConcreteAlternative) { if (types.size() != 1) { throw std::runtime_error("Invalid native type: ConcreteAlternative needs exactly 1 type."); diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index 4818c16ae..dd322dded 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -109,6 +109,10 @@ class Value : public Type { static Type* Make(Instance i); + void initializeDuringDeserialization(Instance i) { + mInstance = i; + } + void postInitializeConcrete() {} Type* cloneForForwardResolutionConcrete() { diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index 47dbc37ca..b8bac0fba 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -202,7 +202,10 @@ def getCInstMeth(self): assert C2.getCInst is not C.getCInst c2Inst = C2snapshot.cls_dict.byKey['getCInst'].func_closure.elements[0].cell_contents.pyobj - c2InstMeth = C2snapshot.cls_dict.byKey['getCInstMeth'].func_closure.elements[0].cell_contents.pyobj + c2InstMethSnap = C2snapshot.cls_dict.byKey['getCInstMeth'].func_closure.elements[0].cell_contents + c2InstMeth = c2InstMethSnap.pyobj + + assert c2InstMethSnap.meth_func.func_closure.elements[0].cell_contents.pyobj is c2InstMeth assert C2().getCInst() is c2Inst assert C2().getCInstMeth() is c2InstMeth From f1657f046c7da231de9b0bfd02f0fb60df1c45cc Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Sun, 30 Jul 2023 01:00:41 +0000 Subject: [PATCH 68/83] Rehydrating more types. --- typed_python/AlternativeType.hpp | 4 + typed_python/PyObjRehydrator.cpp | 158 +++++++++++++++++++++++++++++++ typed_python/PyObjRehydrator.hpp | 12 +++ typed_python/PyObjSnapshot.cpp | 18 +++- 4 files changed, 191 insertions(+), 1 deletion(-) diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 2ec315a82..c30f9bd48 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -418,6 +418,10 @@ class Alternative : public Type { return m_methods; } + const std::vector& getSubtypesConcrete() const { + return m_subtypes_concrete; + } + ConcreteAlternative* concreteSubtype(size_t which); private: diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index 8d08163ff..e08fe6ca0 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -58,6 +58,60 @@ PyObject* PyObjRehydrator::pyobjFor(PyObjSnapshot* snapshot) { } +void PyObjRehydrator::getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& outTypes +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto e: snapshot->mElements) { + Type* t = typeFor(e); + + if (!t) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected types"); + } + + outTypes.push_back(t); + } +} + +void PyObjRehydrator::getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::map& outTypes +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto nameAndType: snapshot->mElements) { + Type* t = typeFor(nameAndType.second); + + if (!t) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected types"); + } + if (!t->isFunction()) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a Function"); + } + + outTypes[nameAndType.first] = t; + } +} + + PyObject* PyObjRehydrator::getNamedElementPyobj( PyObjSnapshot* snapshot, std::string name, @@ -218,6 +272,110 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { return; } + if (snap->mKind == PyObjSnapshot::Kind::BoundMethodType) { + snap->mType = new BoundMethod(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + if (snap->mNames.size() != 1) { + throw std::runtime_error("Corrupt PyObjSnapshot::BoundMethod"); + } + + ((BoundMethod*)snap->mType)->initializeDuringDeserialization( + snap->mNames[0], + getNamedElementType(snap, "self_type") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::TypedCellType) { + snap->mType = new TypedCellType(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + ((TypedCellType*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "element_type") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ForwardType) { + snap->mType = new Forward(snap->mName); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + if (snap->mNamedElements.find("fwd_cell_or_dict") != snap->mNamedElements.end()) { + ((Forward*)snap->mType)->setCellOrDict( + getNamedElementPyobj(snap, "fwd_cell_or_dict") + ); + } + + if (snap->mNamedElements.find("fwd_target") != snap->mNamedElements.end()) { + ((Forward*)snap->mType)->define( + getNamedElementType(snap, "fwd_target") + ); + } + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ConcreteAlternativeType) { + snap->mType = new ConcreteAlternative(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + if (!getNamedElementType(snap, "alternative")->isAlternative()){ + throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); + } + + ((ConcreteAlternative*)snap->mType)->initializeDuringDeserialization( + snap->mNamedInts["which"], + (Alternative*)getNamedElementType(snap, "alternative") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::AlternativeMatcherType) { + snap->mType = new AlternativeMatcher(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + ((AlternativeMatcher*)snap->mType)->initializeDuringDeserialization( + getNamedElementType(snap, "alternative") + ); + + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::AlternativeType) { + snap->mType = new Alternative(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + std::vector > types; + std::map methods; + std::vector subtypesConcrete; + + for (long k = 0; k < snap->mElements.size() && k < snap->mNames.size(); k++) { + types.push_back( + std::make_pair( + typeFor(snap->mElements[k]), + snap->mNames[k] + ) + ); + } + + getNamedBundle(snap, "alt_methods", methods); + getNamedBundle(snap, "alt_subtypes", subtypesConcrete); + + ((Alternative*)snap->mType)->initializeDuringDeserialization( + snap->mName, + snap->mModuleName, + types, + methods, + subtypesConcrete + ); + + return; + } + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); } diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp index c3e812ad0..7fbfaa657 100644 --- a/typed_python/PyObjRehydrator.hpp +++ b/typed_python/PyObjRehydrator.hpp @@ -42,6 +42,18 @@ class PyObjRehydrator { bool allowEmpty=false ); + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& outTypes + ); + + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::map& outTypes + ); + private: void rehydrate(PyObjSnapshot* snapshot); diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 1c0beefbc..6cdf11884 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -596,7 +596,7 @@ void PyObjSnapshot::becomeInternalizedOf( if (t->isBoundMethod()) { mKind = Kind::BoundMethodType; - mNamedElements["element_type"] = maker.internalize( + mNamedElements["self_type"] = maker.internalize( ((BoundMethod*)t)->getFirstArgType() ); mNames.push_back(((BoundMethod*)t)->getFuncName()); @@ -633,6 +633,7 @@ void PyObjSnapshot::becomeInternalizedOf( } mNamedElements["alt_methods"] = maker.internalize(alt->getMethods()); + mNamedElements["alt_subtypes"] = maker.internalize(alt->getSubtypesConcrete()); return; } @@ -678,6 +679,21 @@ void PyObjSnapshot::becomeInternalizedOf( return; } + if (t->isForward()) { + mKind = Kind::ForwardType; + Forward* f = (Forward*)t; + + if (f->getTarget()) { + mNamedElements["fwd_target"] = maker.internalize(f->getTarget()); + } + if (f->getCellOrDict()) { + mNamedElements["fwd_cell_or_dict"] = maker.internalize(f->getCellOrDict()); + } + + mName = f->name(); + return; + } + if (t->isRegister() || t->isString() || t->isBytes() From 8b10e68f37f8f2d663ec385f846306132758f7e3 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 31 Jul 2023 03:01:38 +0000 Subject: [PATCH 69/83] almost done rehydrating types. --- typed_python/PyObjRehydrator.cpp | 147 ++++++++++++++++++++++++++++++- typed_python/PyObjRehydrator.hpp | 24 +++++ typed_python/PyObjSnapshot.cpp | 25 +++++- typed_python/PyObjSnapshot.hpp | 2 + 4 files changed, 193 insertions(+), 5 deletions(-) diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index e08fe6ca0..07bf4ae2f 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -58,6 +58,22 @@ PyObject* PyObjRehydrator::pyobjFor(PyObjSnapshot* snapshot) { } +void PyObjRehydrator::getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& outTypes +) { + std::vector types; + getNamedBundle(snapshot, name, types); + + for (auto t: types) { + if (!t->isHeldClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a HeldClass"); + } + outTypes.push_back((HeldClass*)t); + } +} + void PyObjRehydrator::getNamedBundle( PyObjSnapshot* snapshot, std::string name, @@ -83,6 +99,41 @@ void PyObjRehydrator::getNamedBundle( } } +void PyObjRehydrator::getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& outMembers +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto e: snapshot->mElements) { + outMembers.push_back( + MemberDefinition( + e->mName, + getNamedElementType(e, "type"), + getNamedElementInstance(e, "defaultValue"), + e->mNamedInts["isNonempty"] + ) + ); + } +} + +Instance PyObjRehydrator::getNamedElementInstance( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + // TODO: this is just wrong + return snapshot->mInstance; +} + void PyObjRehydrator::getNamedBundle( PyObjSnapshot* snapshot, std::string name, @@ -97,7 +148,7 @@ void PyObjRehydrator::getNamedBundle( throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); } - for (auto nameAndType: snapshot->mElements) { + for (auto nameAndType: snapshot->mNamedElements) { Type* t = typeFor(nameAndType.second); if (!t) { @@ -107,7 +158,32 @@ void PyObjRehydrator::getNamedBundle( throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a Function"); } - outTypes[nameAndType.first] = t; + outTypes[nameAndType.first] = (Function*)t; + } +} + +void PyObjRehydrator::getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::map& outPyobj +) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto nameAndType: snapshot->mNamedElements) { + PyObject* o = pyobjFor(nameAndType.second); + + if (!o) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a pyobj"); + } + + outPyobj[nameAndType.first] = incref(o); } } @@ -354,10 +430,16 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { std::vector subtypesConcrete; for (long k = 0; k < snap->mElements.size() && k < snap->mNames.size(); k++) { + Type* nt = typeFor(snap->mElements[k]); + + if (!nt->isNamedTuple()) { + throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); + } + types.push_back( std::make_pair( - typeFor(snap->mElements[k]), - snap->mNames[k] + snap->mNames[k], + (NamedTuple*)nt ) ); } @@ -376,6 +458,63 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { return; } + if (snap->mKind == PyObjSnapshot::Kind::ClassType) { + snap->mType = new Class(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + Type* heldClass = getNamedElementType(snap, "held_class_type"); + + if (!heldClass->isHeldClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot.Class"); + } + + ((Class*)snap->mType)->initializeDuringDeserialization( + snap->mName, + (HeldClass*)heldClass + ); + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::HeldClassType) { + snap->mType = new HeldClass(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + std::vector bases; + std::vector members; + std::map memberFunctions; + std::map staticFunctions; + std::map propertyFunctions; + std::map classMembers; + std::map classMethods; + + getNamedBundle(snap, "cls_bases", bases); + getNamedBundle(snap, "cls_members", members); + getNamedBundle(snap, "cls_methods", memberFunctions); + getNamedBundle(snap, "cls_staticmethods", staticFunctions); + getNamedBundle(snap, "cls_properties", propertyFunctions); + getNamedBundle(snap, "cls_classmembers", classMembers); + getNamedBundle(snap, "cls_classmethods", classMethods); + + Type* clsType = getNamedElementType(snap, "cls_type"); + if (!clsType->isClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot.HeldClass"); + } + + ((HeldClass*)snap->mType)->initializeDuringDeserialization( + snap->mName, + bases, + snap->mNamedInts["cls_is_final"], + members, + memberFunctions, + staticFunctions, + propertyFunctions, + classMembers, + classMethods, + (Class*)clsType + ); + return; + } + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); } diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp index 7fbfaa657..9672f3d4e 100644 --- a/typed_python/PyObjRehydrator.hpp +++ b/typed_python/PyObjRehydrator.hpp @@ -42,18 +42,42 @@ class PyObjRehydrator { bool allowEmpty=false ); + Instance getNamedElementInstance( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty=false + ); + void getNamedBundle( PyObjSnapshot* snapshot, std::string name, std::vector& outTypes ); + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& out + ); + + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::vector& outTypes + ); + void getNamedBundle( PyObjSnapshot* snapshot, std::string name, std::map& outTypes ); + void getNamedBundle( + PyObjSnapshot* snapshot, + std::string name, + std::map& outPyobj + ); + private: void rehydrate(PyObjSnapshot* snapshot); diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 6cdf11884..ca6532b94 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -270,6 +270,28 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { return res; } +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); if (mGraph) { @@ -652,7 +674,8 @@ void PyObjSnapshot::becomeInternalizedOf( mName = cls->name(); mModuleName = cls->moduleName(); - mNamedElements["class_type"] = maker.internalize(cls->getClassType()); + mNamedElements["cls_type"] = maker.internalize(cls->getClassType()); + mNamedElements["cls_bases"] = maker.internalize(cls->getBases()); mNamedElements["cls_methods"] = maker.internalize(cls->getOwnMemberFunctions()); mNamedElements["cls_staticmethods"] = maker.internalize(cls->getOwnStaticFunctions()); mNamedElements["cls_classmethods"] = maker.internalize(cls->getOwnClassMethods()); diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 0a3bfa291..7570044cc 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -71,6 +71,8 @@ class PyObjSnapshotMaker { PyObjSnapshot* internalize(const ClosureVariableBindingStep& def); PyObjSnapshot* internalize(const std::vector& def); PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); PyObjSnapshot* internalize(const std::vector& inArgs); PyObjSnapshot* internalize(const std::map& inBindings); PyObjSnapshot* internalize(const std::map& inGlobals); From 35ce44eb466ebefdb3584a587cf26b6b11960969 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 31 Jul 2023 15:25:33 +0000 Subject: [PATCH 70/83] type rehydration written --- typed_python/ClosureVariableBinding.hpp | 2 +- typed_python/FunctionArg.hpp | 6 + typed_python/FunctionOverload.hpp | 13 +- typed_python/HeldClassType.hpp | 5 + typed_python/PyObjRehydrator.cpp | 297 +++++++++++++++--------- typed_python/PyObjRehydrator.hpp | 122 ++++++++-- typed_python/PyObjSnapshot.cpp | 8 + typed_python/PyObjSnapshot.hpp | 9 +- 8 files changed, 327 insertions(+), 135 deletions(-) diff --git a/typed_python/ClosureVariableBinding.hpp b/typed_python/ClosureVariableBinding.hpp index acb2f3651..69cf51875 100644 --- a/typed_python/ClosureVariableBinding.hpp +++ b/typed_python/ClosureVariableBinding.hpp @@ -26,13 +26,13 @@ class ClosureVariableBindingStep { ACCESS_CELL = 4 }; +public: ClosureVariableBindingStep() : mKind(BindingType::ACCESS_CELL), mIndexedFieldToAccess(0), mFunctionToBind(nullptr) {} -public: ClosureVariableBindingStep(Type* bindFunction) : mKind(BindingType::FUNCTION), mIndexedFieldToAccess(0), diff --git a/typed_python/FunctionArg.hpp b/typed_python/FunctionArg.hpp index 1015e5d80..4ed1e4ec7 100644 --- a/typed_python/FunctionArg.hpp +++ b/typed_python/FunctionArg.hpp @@ -20,6 +20,12 @@ class FunctionArg { public: + FunctionArg() : + m_typeFilter(nullptr), + m_defaultValue(nullptr) + { + } + FunctionArg(std::string name, Type* typeFilterOrNull, PyObject* defaultValue, bool isStarArg, bool isKwarg) : m_name(name), m_typeFilter(typeFilterOrNull), diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index cb89d4122..5eb20dc74 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -30,6 +30,17 @@ class FunctionOverload { public: + FunctionOverload() : + mFunctionCode(nullptr), + mFunctionDefaults(nullptr), + mFunctionAnnotations(nullptr), + mReturnType(nullptr), + mSignatureFunction(nullptr), + mCachedFunctionObj(nullptr), + mMethodOf(nullptr) + { + } + FunctionOverload( PyObject* pyFuncCode, PyObject* pyFuncDefaults, @@ -37,7 +48,7 @@ class FunctionOverload { const std::map& inGlobals, const std::vector& pyFuncClosureVarnames, const std::vector& globalsInClosureVarnames, - const std::map closureBindings, + const std::map& closureBindings, Type* returnType, PyObject* pySignatureFunction, const std::vector& args, diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 25ea7edeb..43e9a7570 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -70,9 +70,14 @@ class HashConstCharPtr { } }; + // models a single class member definition class MemberDefinition { public: + MemberDefinition() : mType(nullptr) + { + } + MemberDefinition( const std::string& inName, Type* inType, diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index 07bf4ae2f..af1524fdd 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -16,11 +16,11 @@ void PyObjRehydrator::start(PyObjSnapshot* snapshot) { for (auto elt: snap->mElements) { visit(elt); } - + for (auto elt: snap->mKeys) { visit(elt); } - + for (auto& nameAndElt: snap->mNamedElements) { visit(nameAndElt.second); } @@ -57,136 +57,183 @@ PyObject* PyObjRehydrator::pyobjFor(PyObjSnapshot* snapshot) { return snapshot->mPyObject; } - -void PyObjRehydrator::getNamedBundle( +void PyObjRehydrator::getFrom( PyObjSnapshot* snapshot, - std::string name, - std::vector& outTypes + Type*& out ) { - std::vector types; - getNamedBundle(snapshot, name, types); - - for (auto t: types) { - if (!t->isHeldClass()) { - throw std::runtime_error("Corrupt PyObjSnapshot - expected a HeldClass"); - } - outTypes.push_back((HeldClass*)t); + out = typeFor(snapshot); + if (!out) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a type"); } } -void PyObjRehydrator::getNamedBundle( +void PyObjRehydrator::getFrom( PyObjSnapshot* snapshot, - std::string name, - std::vector& outTypes + PyObject*& out ) { - auto it = snapshot->mNamedElements.find(name); - if (it == snapshot->mNamedElements.end()) { - return; + out = pyobjFor(snapshot); + if (!out) { + throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a pyobj"); } +} - if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { - throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + HeldClass*& out +) { + Type* t = typeFor(snapshot); + if (!t->isHeldClass()) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a HeldClass"); } + out = (HeldClass*)t; +} - for (auto e: snapshot->mElements) { - Type* t = typeFor(e); +void PyObjRehydrator::getFrom( + PyObjSnapshot* e, + MemberDefinition& out +) { + out = MemberDefinition( + e->mName, + getNamedElementType(e, "type"), + getNamedElementInstance(e, "defaultValue"), + e->mNamedInts["isNonempty"] + ); +} - if (!t) { - throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected types"); - } +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + FunctionOverload& out +) { + std::map globals; + std::vector pyFuncClosureVarnames; + std::vector globalsInClosureVarnames; + std::map closureBindings; + std::vector args; + + getNamedBundle(snapshot, "func_globals", globals); + getNamedBundle(snapshot, "func_closure_varnames", pyFuncClosureVarnames); + getNamedBundle(snapshot, "func_globals_in_closure_varnames", globalsInClosureVarnames); + getNamedBundle(snapshot, "func_closure_variable_bindings", closureBindings); + getNamedBundle(snapshot, "func_args", args); + + out = FunctionOverload( + getNamedElementPyobj(snapshot, "func_globals"), + getNamedElementPyobj(snapshot, "func_code"), + getNamedElementPyobj(snapshot, "func_defaults", true), + globals, + pyFuncClosureVarnames, + globalsInClosureVarnames, + closureBindings, + getNamedElementType(snapshot, "func_ret_type", true), + getNamedElementPyobj(snapshot, "func_signature_func", true), + args, + getNamedElementType(snapshot, "func_method_of", true) + ); +} - outTypes.push_back(t); +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + std::string& out +) { + if (!snapshot || snapshot->mKind != PyObjSnapshot::Kind::String) { + throw std::runtime_error("Corrupt PyObjSnapshot.String"); } + + out = snapshot->getStringValue(); } -void PyObjRehydrator::getNamedBundle( +void PyObjRehydrator::getFrom( PyObjSnapshot* snapshot, - std::string name, - std::vector& outMembers + FunctionGlobal& out ) { - auto it = snapshot->mNamedElements.find(name); - if (it == snapshot->mNamedElements.end()) { + if (snapshot->mNamedElements.find("moduleDict") != snapshot->mNamedElements.end() && snapshot->mModuleName.size()) { + out = FunctionGlobal::NamedModuleMember( + getNamedElementPyobj(snapshot, "moduleDict"), + snapshot->mModuleName, + snapshot->mName + ); return; } - if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { - throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + if (snapshot->mNamedElements.find("moduleDict") != snapshot->mNamedElements.end()) { + out = FunctionGlobal::GlobalInDict( + getNamedElementPyobj(snapshot, "moduleDict"), + snapshot->mName + ); + return; } - for (auto e: snapshot->mElements) { - outMembers.push_back( - MemberDefinition( - e->mName, - getNamedElementType(e, "type"), - getNamedElementInstance(e, "defaultValue"), - e->mNamedInts["isNonempty"] - ) + if (snapshot->mNamedElements.find("cell") != snapshot->mNamedElements.end()) { + out = FunctionGlobal::GlobalInCell( + getNamedElementPyobj(snapshot, "cell") ); + return; } } -Instance PyObjRehydrator::getNamedElementInstance( +void PyObjRehydrator::getFrom( PyObjSnapshot* snapshot, - std::string name, - bool allowEmpty + FunctionArg& out ) { - // TODO: this is just wrong - return snapshot->mInstance; + Type* filter = getNamedElementType(snapshot, "arg_type_filter", true); + PyObject* defaultVal = getNamedElementPyobj(snapshot, "arg_default_value", true); + bool isStarArg = snapshot->mNamedInts["arg_is_star_arg"]; + bool isKwarg = snapshot->mNamedInts["arg_is_kwarg"]; + + out = FunctionArg( + snapshot->mName, + filter, + defaultVal, + isStarArg, + isKwarg + ); } -void PyObjRehydrator::getNamedBundle( +void PyObjRehydrator::getFrom( PyObjSnapshot* snapshot, - std::string name, - std::map& outTypes + ClosureVariableBinding& out ) { - auto it = snapshot->mNamedElements.find(name); - if (it == snapshot->mNamedElements.end()) { - return; - } + std::vector steps; - if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { - throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + for (auto e: snapshot->mElements) { + ClosureVariableBindingStep step; + getFrom(e, step); + steps.push_back(step); } - for (auto nameAndType: snapshot->mNamedElements) { - Type* t = typeFor(nameAndType.second); - - if (!t) { - throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected types"); - } - if (!t->isFunction()) { - throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a Function"); - } - - outTypes[nameAndType.first] = (Function*)t; - } + out = ClosureVariableBinding(steps); } -void PyObjRehydrator::getNamedBundle( +void PyObjRehydrator::getFrom( PyObjSnapshot* snapshot, - std::string name, - std::map& outPyobj + ClosureVariableBindingStep& out ) { - auto it = snapshot->mNamedElements.find(name); - if (it == snapshot->mNamedElements.end()) { + if (snapshot->mNamedElements.find("step_func") != snapshot->mNamedElements.end()) { + Function* f; + getFrom(snapshot->mNamedElements.find("step_func")->second, f); + out = ClosureVariableBindingStep(f); return; } - if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { - throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + if (snapshot->mNames.size()) { + out = ClosureVariableBindingStep(snapshot->mNames[0]); + return; } - for (auto nameAndType: snapshot->mNamedElements) { - PyObject* o = pyobjFor(nameAndType.second); - - if (!o) { - throw std::runtime_error("Corrupt PyObjSnapshot.InternalBundle - expected a pyobj"); - } - - outPyobj[nameAndType.first] = incref(o); + if (snapshot->mNamedInts.find("step_index") != snapshot->mNamedInts.end()) { + out = ClosureVariableBindingStep(snapshot->mNamedInts["step_index"]); + return; } } +Instance PyObjRehydrator::getNamedElementInstance( + PyObjSnapshot* snapshot, + std::string name, + bool allowEmpty +) { + // TODO: this is just wrong + return snapshot->mInstance; +} PyObject* PyObjRehydrator::getNamedElementPyobj( PyObjSnapshot* snapshot, @@ -300,7 +347,7 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { rehydrate(snap->mNamedElements["value_instance"]); ((Value*)snap->mType)->initializeDuringDeserialization( - snap->mNamedElements["value_instance"]->mInstance + snap->mNamedElements["value_instance"]->mInstance ); return; @@ -309,49 +356,49 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { if (snap->mKind == PyObjSnapshot::Kind::OneOfType) { snap->mType = new OneOfType(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + std::vector types; for (auto s: snap->mElements) { types.push_back(typeFor(s)); } ((OneOfType*)snap->mType)->initializeDuringDeserialization(types); - + return; } if (snap->mKind == PyObjSnapshot::Kind::TupleType) { snap->mType = new Tuple(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + std::vector types; for (auto s: snap->mElements) { types.push_back(typeFor(s)); } ((Tuple*)snap->mType)->initializeDuringDeserialization(types); - + return; } if (snap->mKind == PyObjSnapshot::Kind::NamedTupleType) { snap->mType = new NamedTuple(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + std::vector types; for (auto s: snap->mElements) { types.push_back(typeFor(s)); } ((NamedTuple*)snap->mType)->initializeDuringDeserialization(types, snap->mNames); - + return; } if (snap->mKind == PyObjSnapshot::Kind::BoundMethodType) { snap->mType = new BoundMethod(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + if (snap->mNames.size() != 1) { throw std::runtime_error("Corrupt PyObjSnapshot::BoundMethod"); } @@ -360,25 +407,25 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { snap->mNames[0], getNamedElementType(snap, "self_type") ); - + return; } if (snap->mKind == PyObjSnapshot::Kind::TypedCellType) { snap->mType = new TypedCellType(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + ((TypedCellType*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "element_type") ); - + return; } if (snap->mKind == PyObjSnapshot::Kind::ForwardType) { snap->mType = new Forward(snap->mName); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + if (snap->mNamedElements.find("fwd_cell_or_dict") != snap->mNamedElements.end()) { ((Forward*)snap->mType)->setCellOrDict( getNamedElementPyobj(snap, "fwd_cell_or_dict") @@ -390,14 +437,14 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { getNamedElementType(snap, "fwd_target") ); } - + return; } if (snap->mKind == PyObjSnapshot::Kind::ConcreteAlternativeType) { snap->mType = new ConcreteAlternative(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + if (!getNamedElementType(snap, "alternative")->isAlternative()){ throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); } @@ -406,32 +453,32 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { snap->mNamedInts["which"], (Alternative*)getNamedElementType(snap, "alternative") ); - + return; } if (snap->mKind == PyObjSnapshot::Kind::AlternativeMatcherType) { snap->mType = new AlternativeMatcher(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + ((AlternativeMatcher*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "alternative") ); - + return; } if (snap->mKind == PyObjSnapshot::Kind::AlternativeType) { snap->mType = new Alternative(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + std::vector > types; std::map methods; std::vector subtypesConcrete; for (long k = 0; k < snap->mElements.size() && k < snap->mNames.size(); k++) { Type* nt = typeFor(snap->mElements[k]); - + if (!nt->isNamedTuple()) { throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); } @@ -454,23 +501,23 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { methods, subtypesConcrete ); - + return; } if (snap->mKind == PyObjSnapshot::Kind::ClassType) { snap->mType = new Class(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - + Type* heldClass = getNamedElementType(snap, "held_class_type"); - + if (!heldClass->isHeldClass()) { throw std::runtime_error("Corrupt PyObjSnapshot.Class"); } ((Class*)snap->mType)->initializeDuringDeserialization( snap->mName, - (HeldClass*)heldClass + (HeldClass*)heldClass ); return; } @@ -515,6 +562,32 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { return; } + if (snap->mKind == PyObjSnapshot::Kind::FunctionType) { + std::string name = snap->mName; + std::string qualname = snap->mQualname; + std::string moduleName = snap->mModuleName; + std::vector overloads; + Type* closureType = getNamedElementType(snap, "closure_type"); + bool isEntrypoint = snap->mNamedInts["is_nocompile"]; + bool isNocompile = snap->mNamedInts["is_nocompile"]; + + getNamedBundle(snap, "func_overloads", overloads); + + snap->mType = new Function(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + ((Function*)snap->mType)->initializeDuringDeserialization( + name, + qualname, + moduleName, + overloads, + closureType, + isEntrypoint, + isNocompile + ); + return; + } + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); } @@ -796,7 +869,7 @@ void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { PyObjSnapshot* clsDictPyObj = obj->mNamedElements["cls_dict"]; - for (long k = 0; k < clsDictPyObj->elements().size() + for (long k = 0; k < clsDictPyObj->elements().size() && k < clsDictPyObj->keys().size(); k++) { if (clsDictPyObj->keys()[k]->isString()) { PyObject_SetAttrString( @@ -806,9 +879,9 @@ void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { ); } } - } else + } else if (obj->mKind == PyObjSnapshot::Kind::PyDict || obj->mKind == PyObjSnapshot::Kind::PyClassDict) { - for (long k = 0; k < obj->mElements.size() + for (long k = 0; k < obj->mElements.size() && k < obj->mKeys.size(); k++) { PyDict_SetItem( obj->mPyObject, @@ -816,7 +889,7 @@ void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { obj->mElements[k]->getPyObj() ); } - } else + } else if (obj->mKind == PyObjSnapshot::Kind::PyCell) { auto it = obj->mNamedElements.find("cell_contents"); if (it != obj->mNamedElements.end()) { @@ -825,7 +898,7 @@ void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { it->second->getPyObj() ); } - } else + } else if (obj->mType) { obj->mType->recomputeName(); obj->mType->finalizeType(); diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp index 9672f3d4e..a48e85a77 100644 --- a/typed_python/PyObjRehydrator.hpp +++ b/typed_python/PyObjRehydrator.hpp @@ -17,8 +17,10 @@ #pragma once #include +#include "PyObjSnapshot.hpp" -class PyObjSnapshot; +class FunctionOverload; +class FunctionGlobal; class PyObjRehydrator { public: @@ -48,37 +50,129 @@ class PyObjRehydrator { bool allowEmpty=false ); - void getNamedBundle( + void getFrom( PyObjSnapshot* snapshot, - std::string name, - std::vector& outTypes + Type*& out ); - void getNamedBundle( + void getFrom( PyObjSnapshot* snapshot, - std::string name, - std::vector& out + PyObject*& out ); - void getNamedBundle( + void getFrom( PyObjSnapshot* snapshot, - std::string name, - std::vector& outTypes + Function*& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + HeldClass*& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + MemberDefinition& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + FunctionOverload& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + std::string& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + FunctionGlobal& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + FunctionArg& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + ClosureVariableBinding& out + ); + + void getFrom( + PyObjSnapshot* snapshot, + ClosureVariableBindingStep& out ); + template void getNamedBundle( PyObjSnapshot* snapshot, std::string name, - std::map& outTypes - ); + std::map& out + ) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto nameAndType: it->second->mNamedElements) { + T item; + + getFrom(nameAndType.second, item); + + out[nameAndType.first] = item; + } + } + template void getNamedBundle( PyObjSnapshot* snapshot, std::string name, - std::map& outPyobj - ); + std::vector& out + ) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + for (auto snap: it->second->mElements) { + T item; + + getFrom(snap, item); + + out.push_back(item); + } + } + + template + void getNamed( + PyObjSnapshot* snapshot, + std::string name, + T& out + ) { + auto it = snapshot->mNamedElements.find(name); + if (it == snapshot->mNamedElements.end()) { + return; + } + + if (it->second->mKind != PyObjSnapshot::Kind::InternalBundle) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a bundle"); + } + + getFrom(it->second, out); + } private: + void rehydrate(PyObjSnapshot* snapshot); void rehydrateTpType(PyObjSnapshot* snap); diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index ca6532b94..19ad4211d 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -1,7 +1,15 @@ #include "PyObjSnapshot.hpp" #include "PyObjGraphSnapshot.hpp" +#include "PyObjRehydrator.hpp" +void PyObjSnapshot::rehydrate() { + PyObjRehydrator rehydrator; + + rehydrator.start(this); + rehydrator.finalize(); +} + std::string PyObjSnapshot::kindAsString() const { if (mKind == Kind::Uninitialized) { return "Uninitialized"; diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 7570044cc..0e5ebbc89 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -20,12 +20,12 @@ #include "Type.hpp" #include "Instance.hpp" #include "PythonTypeInternals.hpp" -#include "PyObjRehydrator.hpp" class PyObjGraphSnapshot; class PyObjSnapshot; class FunctionGlobal; class FunctionOverload; +class PyObjRehydrator; /********************************* PyObjSnapshot @@ -562,12 +562,7 @@ class PyObjSnapshot { return mPyObject; } - void rehydrate() { - PyObjRehydrator rehydrator; - - rehydrator.start(this); - rehydrator.finalize(); - } + void rehydrate(); std::string toString() const { return "PyObjSnapshot." + kindAsString() + "()"; From 8280ff0353ce93d7a72a0c1fa6c1ac84e7fa9a15 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 31 Jul 2023 18:42:47 +0000 Subject: [PATCH 71/83] Implement sha-hashing for PyObjSnapshot graphs. --- typed_python/PyObjGraphSnapshot.cpp | 235 ++++++++++++++++++++++ typed_python/PyObjGraphSnapshot.hpp | 41 +++- typed_python/PyObjRehydrator.cpp | 11 + typed_python/PyObjSnapshot.hpp | 49 ++++- typed_python/PyObjSnapshotGroupSearch.hpp | 130 ++++++++++++ typed_python/PyPyObjSnapshot.cpp | 43 ++-- typed_python/ShaHash.hpp | 16 ++ typed_python/Type.cpp | 2 +- typed_python/all.cpp | 1 + typed_python/py_obj_snapshot_test.py | 30 ++- 10 files changed, 524 insertions(+), 34 deletions(-) create mode 100644 typed_python/PyObjGraphSnapshot.cpp create mode 100644 typed_python/PyObjSnapshotGroupSearch.hpp diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp new file mode 100644 index 000000000..4bb0de006 --- /dev/null +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -0,0 +1,235 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#include "PyObjGraphSnapshot.hpp" +#include "PyObjSnapshotGroupSearch.hpp" + +template +ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { + if (snap->getKind() == PyObjSnapshot::Kind::Instance) { + if (snap->getInstance().type()->isPOD()) { + return ShaHash(int(snap->getKind())) + + ShaHash::SHA1(snap->getInstance().data(), snap->getInstance().type()->bytecount()); + } + throw std::runtime_error( + "Can't hash a PyObjSnapshot Instance of type " + snap->getInstance().type()->name() + ); + } + if (snap->getKind() == PyObjSnapshot::Kind::PrimitiveType) { + return ShaHash(int(snap->getKind())) + ShaHash(int(snap->getType()->getTypeCategory())); + } + if (snap->getKind() == PyObjSnapshot::Kind::ArbitraryPyObject) { + // probably we should try to serialize it? + throw std::runtime_error( + "Can't hash a PyObjSnapshot.ArbitraryPyObject of type " + + std::string(snap->getPyObj()->ob_type->tp_name) + ); + } + + ShaHash res(int(snap->getKind())); + + if (snap->getStringValue().size()) { + res = res + ShaHash(1) + ShaHash(snap->getStringValue()); + } + if (snap->getName().size()) { + res = res + ShaHash(2) + ShaHash(snap->getName()); + } + if (snap->getModuleName().size()) { + res = res + ShaHash(3) + ShaHash(snap->getModuleName()); + } + if (snap->getQualname().size()) { + res = res + ShaHash(4) + ShaHash(snap->getQualname()); + } + if (snap->names().size()) { + res = res + ShaHash(6) + ShaHash(snap->names().size()); + + for (auto n: snap->names()) { + res = res + ShaHash(n); + } + } + if (snap->elements().size()) { + res = res + ShaHash(7) + ShaHash(snap->elements().size()); + + for (auto n: snap->elements()) { + res = res + compute(n); + } + } + if (snap->keys().size()) { + res = res + ShaHash(8) + ShaHash(snap->keys().size()); + + for (auto n: snap->keys()) { + res = res + compute(n); + } + } + if (snap->namedElements().size()) { + res = res + ShaHash(10) + ShaHash(snap->namedElements().size()); + + for (auto& nameAndElt: snap->namedElements()) { + res = res + ShaHash(nameAndElt.first) + compute(nameAndElt.second); + } + } + if (snap->namedInts().size()) { + res = res + ShaHash(11) + ShaHash(snap->namedInts().size()); + + for (auto& nameAndElt: snap->namedInts()) { + res = res + ShaHash(nameAndElt.first) + ShaHash(nameAndElt.second); + } + } + + return res; +} + + +ShaHash PyObjGraphSnapshot::hashFor(PyObjSnapshot* snap) { + if (snap->getGraph() != this) { + throw std::runtime_error("Can't graph a hash of a PyObjSnapshot we don't have"); + } + + auto it = mSnapToHash.find(snap); + if (it != mSnapToHash.end()) { + return it->second; + } + + mGroupSearch.add(snap); + + while (mGroupsConsumed < mGroupSearch.getGroups().size()) { + computeHashesFor(*mGroupSearch.getGroups()[mGroupsConsumed++]); + } + + it = mSnapToHash.find(snap); + + if (it != mSnapToHash.end()) { + return it->second; + } + + throw std::runtime_error("PyObjGraphSnapshot failed to produce a hash for one of its objects."); +} + +void PyObjGraphSnapshot::computeHashesFor(const std::unordered_set& group) { + PyObjSnapshot* root = pickRootFor(group); + + // order our snapshots lexically + std::unordered_map snapsSeen; + std::vector snapsInOrder; + + std::function visit = [&](PyObjSnapshot* s) { + // if its not in the group, ignore it + if (group.find(s) == group.end()) { + return; + } + + // if we saw it already, skip it + if (snapsSeen.find(s) != snapsSeen.end()) { + return; + } + + // record it + int snapIx = snapsSeen.size(); + snapsSeen[s] = snapIx; + snapsInOrder.push_back(s); + + // visit our outbound edges + s->visitOutbound(visit); + }; + + visit(root); + + if (snapsSeen.size() != group.size()) { + throw std::runtime_error("PyObjGraphSnapshot::computeHashFor failed to find all snaps"); + } + + ShaHash groupHash = ShaHash(snapsInOrder.size()); + + for (long i = 0; i < snapsInOrder.size(); i++) { + ShaHash thisSnapHash = computeHashFor( + snapsInOrder[i], + [&](PyObjSnapshot* snap) { + auto it = snapsSeen.find(snap); + if (it != snapsSeen.end()) { + return ShaHash(1) + ShaHash(it->second); + } + auto it2 = mSnapToHash.find(snap); + if (it2 != mSnapToHash.end()) { + return ShaHash(2) + it2->second; + } + + throw std::runtime_error("PyObjGraphSnapshot::computeHashFor failed to find a snap."); + } + ); + + groupHash = groupHash + ShaHash(i) + thisSnapHash; + } + + for (long i = 0; i < snapsInOrder.size(); i++) { + installSnapHash(snapsInOrder[i], groupHash + ShaHash(i)); + } +} + +void PyObjGraphSnapshot::installSnapHash(PyObjSnapshot* snap, ShaHash h) { + mSnapToHash[snap] = h; + + if (!mHashToSnap[h]) { + mHashToSnap[h] = snap; + } else { + // this can happen any time in a single graph we have multiple copies of objects that + // look the same (as far as what can be seen from their references). As an example, + // the empty tuple or empty list will always be the same hash in any graph + mRedundantSnaps.insert(snap); + } +} + +PyObjSnapshot* PyObjGraphSnapshot::pickRootFor(const std::unordered_set& group) { + // pick a 'root' node by computing a shallow hash and taking the first value for which + // there is only one hash + std::unordered_map curHashes; + std::unordered_map newHashes; + long passes = 0; + + while (passes < group.size() + 1) { + for (auto s: group) { + ShaHash h = computeHashFor(s, [&](PyObjSnapshot* edge) { + if (group.find(edge) == group.end()) { + auto it = mSnapToHash.find(edge); + if (it != mSnapToHash.end()) { + return it->second; + } + + throw std::runtime_error("Found a PyObjSnapshot that's not in our graph"); + } + + return curHashes[s]; + }); + + newHashes[s] = h; + } + + // find the first hash that's unique + std::map > toHash; + for (auto snapAndHash: newHashes) { + toHash[snapAndHash.second].insert(snapAndHash.first); + } + + for (auto& hashAndObj: toHash) { + if (hashAndObj.second.size() == 1) { + return *hashAndObj.second.begin(); + } + } + } + + // we didn't find one even after N passes, so our objects look identical. + // just pick one. + return *group.begin(); +} diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index 98a17e37a..0985677c1 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -17,7 +17,8 @@ #pragma once #include - +#include "PyObjSnapshotGroupSearch.hpp" +#include "ShaHash.hpp" class PyObjSnapshot; @@ -25,15 +26,24 @@ class PyObjSnapshot; /********************************* PyObjGraphSnapshot -Holds a collection of PyObjSnapshot objects. When this dies, they'll die. +Holds a collection of PyObjSnapshot objects. This graph object owns its snapshots +and frees them when it iself is release. + +There's a global PyObjGraphSnapshot that holds all the interned objects accessible from + + PyObjGraphSnapshot::internal() -There's a global PyObjGraphSnapshot that holds all the interned objects. +A graph knows how to assign a unique hash to every object it contains. **********************************/ class PyObjGraphSnapshot { public: - PyObjGraphSnapshot() {} + PyObjGraphSnapshot() : + mGroupsConsumed(0), + mGroupSearch(this) + { + } ~PyObjGraphSnapshot() { for (auto oPtr: mObjects) { @@ -60,6 +70,29 @@ class PyObjGraphSnapshot { return mObjects; } + ShaHash hashFor(PyObjSnapshot* snap); + private: + void installSnapHash(PyObjSnapshot* snap, ShaHash h); + + void computeHashesFor(const std::unordered_set& group); + + PyObjSnapshot* pickRootFor(const std::unordered_set& group); + + // all of our objects std::unordered_set mObjects; + + // for each snapshot, a hash + std::unordered_map mSnapToHash; + + // for each hash, the object that matches it. It's possible to produce objects in a graph + // that are indistinguishable from each other, in which case we simply pick one to go + // 'first' and include that extra information in the hash. + std::unordered_map mHashToSnap; + + // snaps with the same sha hash as each other + std::unordered_set mRedundantSnaps; + + PyObjSnapshotGroupSearch mGroupSearch; + long mGroupsConsumed; }; diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index af1524fdd..9a964ee8e 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -88,6 +88,17 @@ void PyObjRehydrator::getFrom( out = (HeldClass*)t; } +void PyObjRehydrator::getFrom( + PyObjSnapshot* snapshot, + Function*& out +) { + Type* t = typeFor(snapshot); + if (!t->isFunction()) { + throw std::runtime_error("Corrupt PyObjSnapshot - expected a Function"); + } + out = (Function*)t; +} + void PyObjRehydrator::getFrom( PyObjSnapshot* e, MemberDefinition& out diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 0e5ebbc89..adbc88fd4 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -113,6 +113,15 @@ class PyObjSnapshot { friend class PyObjSnapshotMaker; friend class PyObjRehydrator; + PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr) : + mKind(Kind::Uninitialized), + mType(nullptr), + mPyObject(nullptr), + mGraph(inGraph) + { + } + +public: enum class Kind { // this should never be visible in a running program Uninitialized = 0, @@ -224,15 +233,6 @@ class PyObjSnapshot { InternalBundle }; - PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr) : - mKind(Kind::Uninitialized), - mType(nullptr), - mPyObject(nullptr), - mGraph(inGraph) - { - } - -public: ~PyObjSnapshot() { decref(mPyObject); } @@ -246,6 +246,10 @@ class PyObjSnapshot { return res; } + Kind getKind() const { + return mKind; + } + PyObjGraphSnapshot* getGraph() const { return mGraph; } @@ -414,6 +418,10 @@ class PyObjSnapshot { return mNamedElements; } + const std::vector& names() const { + return mNames; + } + const std::map& namedInts() const { return mNamedInts; } @@ -430,6 +438,10 @@ class PyObjSnapshot { return mModuleName; } + const std::string& getQualname() const { + return mQualname; + } + template void _visitReferencedTypes(const visitor_type& v) { if (mKind == Kind::PrimitiveType) { @@ -496,6 +508,10 @@ class PyObjSnapshot { ::Type* getType() { + if (mType) { + return mType; + } + if (mKind == Kind::PrimitiveType) { return mType; } @@ -691,6 +707,21 @@ class PyObjSnapshot { return result; } + template + void visitOutbound(const visitor_type& v) { + for (auto e: mElements) { + v(e); + } + + for (auto e: mKeys) { + v(e); + } + + for (auto& nameAndE: mNamedElements) { + v(nameAndE.second); + } + } + private: // ensure we won't crash if we interact with this object. void validateAfterDeserialization() { diff --git a/typed_python/PyObjSnapshotGroupSearch.hpp b/typed_python/PyObjSnapshotGroupSearch.hpp new file mode 100644 index 000000000..2157c8300 --- /dev/null +++ b/typed_python/PyObjSnapshotGroupSearch.hpp @@ -0,0 +1,130 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include "PyObjSnapshot.hpp" + +// find strongly-connected groups of python objects and Type objects (as far as the +// compiler is concerned) +class PyObjSnapshotGroupSearch { +public: + PyObjSnapshotGroupSearch(PyObjGraphSnapshot* graph) : mGraph(graph) + { + } + + void add(PyObjSnapshot* o) { + pushGroup(o); + findAllGroups(); + } + + const std::vector > >& getGroups() const { + return mOutGroups; + } + +private: + void pushGroup(PyObjSnapshot* o) { + if (mInAGroup.find(o) != mInAGroup.end()) { + return; + } + + mInAGroup.insert(o); + + mGroups.push_back( + std::shared_ptr >( + new std::unordered_set() + ) + ); + + mGroupOutboundEdges.push_back( + std::shared_ptr >( + new std::vector() + ) + ); + + mGroups.back()->insert(o); + + o->visitOutbound([&](PyObjSnapshot* snap) { + mGroupOutboundEdges.back()->push_back(snap); + }); + } + + void findAllGroups() { + while (mGroups.size()) { + doOneStep(); + } + } + + void doOneStep() { + if (mGroupOutboundEdges.back()->size() == 0) { + // this group is finished.... + mOutGroups.push_back(mGroups.back()); + + for (auto gElt: *mGroups.back()) { + mInAGroup.erase(gElt); + } + + mGroups.pop_back(); + mGroupOutboundEdges.pop_back(); + } else { + // pop an outbound edge and see where does it go? + PyObjSnapshot* o = mGroupOutboundEdges.back()->back(); + + mGroupOutboundEdges.back()->pop_back(); + + if (mInAGroup.find(o) == mInAGroup.end()) { + // this is new + pushGroup(o); + } else { + // this is above us somewhere + while (mGroups.size() && mGroups.back()->find(o) == mGroups.back()->end()) { + // merge these two mGroups together + mGroups[mGroups.size() - 2]->insert( + mGroups.back()->begin(), mGroups.back()->end() + ); + mGroups.pop_back(); + + mGroupOutboundEdges[mGroupOutboundEdges.size() - 2]->insert( + mGroupOutboundEdges[mGroupOutboundEdges.size() - 2]->end(), + mGroupOutboundEdges.back()->begin(), + mGroupOutboundEdges.back()->end() + ); + mGroupOutboundEdges.pop_back(); + } + + if (!mGroups.size()) { + // this shouldn't ever happen, but we want to know if it does + throw std::runtime_error("Never found parent of object!"); + } + } + } + } + +private: + PyObjGraphSnapshot* mGraph; + + // everything in a group somewhere + std::unordered_set mInAGroup; + + // each group + std::vector > > mGroups; + + // for each group, the set of things it reaches out to + std::vector > > mGroupOutboundEdges; + + // the final groups we ended up with + std::vector > > mOutGroups; +}; diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index b0db80e35..c71e247ac 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -104,7 +104,7 @@ PyObject* PyPyObjSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kw } PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { - PyPyObjSnapshot* pyCVPO = (PyPyObjSnapshot*)selfObj; + PyPyObjSnapshot* pySnap = (PyPyObjSnapshot*)selfObj; return translateExceptionToPyObject([&] { if (!PyUnicode_Check(attrName)) { @@ -113,13 +113,13 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { std::string attr(PyUnicode_AsUTF8(attrName)); - PyObjSnapshot* obj = pyCVPO->mPyobj; + PyObjSnapshot* obj = pySnap->mPyobj; auto it = obj->namedElements().find(attr); if (it != obj->namedElements().end()) { return PyPyObjSnapshot::newPyObjSnapshot( it->second, - it->second->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + it->second->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr ); } @@ -141,7 +141,12 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { } if (attr == "graph") { - return incref(pyCVPO->mGraph ? pyCVPO->mGraph : Py_None); + return incref(pySnap->mGraph ? pySnap->mGraph : Py_None); + } + + if (attr == "shaHash" && obj->getGraph()) { + ShaHash hash = obj->getGraph()->hashFor(obj); + return PyUnicode_FromString(hash.digestAsHexString().c_str()); } if (attr == "type" && obj->getType()) { @@ -165,57 +170,57 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { } if (attr == "elements") { - if (!pyCVPO->mElements) { - pyCVPO->mElements = PyList_New(0); + if (!pySnap->mElements) { + pySnap->mElements = PyList_New(0); for (auto p: obj->elements()) { PyList_Append( - pyCVPO->mElements, + pySnap->mElements, PyPyObjSnapshot::newPyObjSnapshot( p, - p->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr ) ); } } - return incref(pyCVPO->mElements); + return incref(pySnap->mElements); } if (attr == "keys") { - if (!pyCVPO->mKeys) { - pyCVPO->mKeys = PyList_New(0); + if (!pySnap->mKeys) { + pySnap->mKeys = PyList_New(0); for (auto p: obj->keys()) { PyList_Append( - pyCVPO->mKeys, + pySnap->mKeys, PyPyObjSnapshot::newPyObjSnapshot( p, - p->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr ) ); } } - return incref(pyCVPO->mKeys); + return incref(pySnap->mKeys); } if (attr == "byKey") { - if (!pyCVPO->mByKey) { - pyCVPO->mByKey = PyDict_New(); + if (!pySnap->mByKey) { + pySnap->mByKey = PyDict_New(); for (long k = 0; k < obj->keys().size(); k++) { PyObjSnapshot* p = obj->elements()[k]; PyDict_SetItem( - pyCVPO->mByKey, + pySnap->mByKey, obj->keys()[k]->getPyObj(), PyPyObjSnapshot::newPyObjSnapshot( p, - p->getGraph() == obj->getGraph() ? pyCVPO->mGraph : nullptr + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr ) ); } } - return incref(pyCVPO->mByKey); + return incref(pySnap->mByKey); } return PyObject_GenericGetAttr(selfObj, attrName); diff --git a/typed_python/ShaHash.hpp b/typed_python/ShaHash.hpp index 237b1a3a5..02a2ead61 100644 --- a/typed_python/ShaHash.hpp +++ b/typed_python/ShaHash.hpp @@ -189,3 +189,19 @@ inline ShaHash operator+(const ShaHash& l, const ShaHash& r) { return tr; } + +namespace std { + template<> + struct hash { + typedef ShaHash argument_type; + typedef std::size_t result_type; + + result_type operator()(argument_type const& s) const noexcept { + return ((size_t)s[0] + (((size_t)s[1]) << 32)) + ^ ((size_t)s[2] + (((size_t)s[3]) << 32)) + ^ (size_t)s[3]; + } + }; +} + + diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 8865b25e9..d01e13cd6 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -1,5 +1,5 @@ /****************************************************************************** - Copyright 2017-2019 typed_python Authors + Copyright 2017-2023 typed_python Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/typed_python/all.cpp b/typed_python/all.cpp index b302539dd..fa1c9c216 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -79,6 +79,7 @@ compile the entire group all at once. #include "PyFunctionOverload.cpp" #include "PyFunctionGlobal.cpp" #include "PyObjSnapshot.cpp" +#include "PyObjGraphSnapshot.cpp" #include "PyObjRehydrator.cpp" #include "PyPyObjSnapshot.cpp" #include "PyPyObjGraphSnapshot.cpp" diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index b8bac0fba..17d1120b5 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -1,7 +1,15 @@ import numpy from typed_python import ListOf -from typed_python._types import PyObjSnapshot +from typed_python._types import PyObjSnapshot, _enableTypeAutoresolution + + +class DisableForwardAutoresolve: + def __enter__(self): + _enableTypeAutoresolution(False) + + def __exit__(self, *args): + _enableTypeAutoresolution(True) def test_make_snapshot_basic(): @@ -223,3 +231,23 @@ def f(): assert s1.kind == 'PyFunction' assert s1.func_closure.graph is s1.graph assert s1.func_closure.graph is not s2.func_closure.graph + + +def test_sha_hashing_of_primitive_types(): + assert PyObjSnapshot.create(int).shaHash != PyObjSnapshot.create(float).shaHash + assert PyObjSnapshot.create((1, 2, 3)).shaHash == PyObjSnapshot.create((1, 2, 3)).shaHash + + +def graphIdempotenceCheck(x): + snap = PyObjSnapshot.create(x, False) + snap2 = PyObjSnapshot.create(snap.pyobj, False) + + assert snap.shaHash == snap2.shaHash + + +def test_rehydration_preserves_sha_hashing(): + # disable forward autoresolution so we can look at the type graphs without + # the autoresolver going in and resolving forwards + with DisableForwardAutoresolve(): + graphIdempotenceCheck(int) + graphIdempotenceCheck(ListOf(int)) From b0c127cf4db6d60c3c54be8615baeb0b7f7837a8 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 31 Jul 2023 21:05:03 +0000 Subject: [PATCH 72/83] FunctionGlobal doesn't have a dependency on PyObjSnapshot anymore. The plan is for PyObjSnapshot to be tooling we use to internalize things in a more rigorous way. --- typed_python/FunctionGlobal.hpp | 90 +++++++++++--------------- typed_python/FunctionOverload.hpp | 4 +- typed_python/FunctionType.hpp | 4 +- typed_python/PyFunctionGlobal.cpp | 2 +- typed_python/Type.cpp | 14 +--- typed_python/type_construction_test.py | 2 +- 6 files changed, 44 insertions(+), 72 deletions(-) diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index ecb8f4d6f..d6d99d32f 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -24,7 +24,7 @@ class FunctionGlobal { enum class GlobalType { Unbound = 1, // this global is not bound to anything - its a Name error NamedModuleMember = 2, // this global is a member of a particular module - Constant = 3, // this global is bound to a constant that we resolved to at some point + Constant = 3, // this global is bound to a constant that we resolved at some point GlobalInDict = 4, // this global is an unresolved variable residing in some python dict GlobalInCell = 5, // this global is an unresolved variable residing in some PyCell }; @@ -34,7 +34,7 @@ class FunctionGlobal { PyObject* dictOrCell, std::string name, std::string moduleName, - PyObjSnapshot* constant + Type* constant ) : mKind(inKind), mModuleDictOrCell(incref(dictOrCell)), @@ -51,6 +51,25 @@ class FunctionGlobal { { } + FunctionGlobal withConstantsInternalized(const std::map& typeMap) { + if (!(isGlobalInDict() || isGlobalInCell())) { + return *this; + } + + Type* t = getValueAsType(); + + if (t) { + auto it = typeMap.find(t); + if (it != typeMap.end()) { + return FunctionGlobal::Constant(it->second); + } else { + return FunctionGlobal::Constant(t); + } + } + + return *this; + } + std::string toString() { if (isUnbound()) { return "FunctionGlobal.Unbound()"; @@ -69,7 +88,7 @@ class FunctionGlobal { } if (isConstant()) { - return "FunctionGlobal.Constant(" + mConstant->toString() + ")"; + return "FunctionGlobal.Constant(type=" + mConstant->name() + ")"; } throw std::runtime_error("Unknown FunctionGlobal Kind"); @@ -79,7 +98,11 @@ class FunctionGlobal { return FunctionGlobal(); } - static FunctionGlobal Constant(PyObjSnapshot* constant) { + static FunctionGlobal Constant(Type* constant) { + if (!constant) { + throw std::runtime_error("FunctionGlobal::Constant can't be null"); + } + return FunctionGlobal( GlobalType::Constant, nullptr, @@ -173,10 +196,7 @@ class FunctionGlobal { } if (isConstant()) { - if (mConstant->isType()) { - return mConstant->getType(); - } - return nullptr; + return mConstant; } throw std::runtime_error("Unknown global kind."); @@ -192,7 +212,7 @@ class FunctionGlobal { } if (isConstant()) { - return mConstant->getPyObj(); + return (PyObject*)PyInstance::typeObj(mConstant); } throw std::runtime_error("Unknown global kind."); @@ -202,7 +222,7 @@ class FunctionGlobal { return mKind; } - PyObjSnapshot* getConstant() const { + Type* getConstant() const { return mConstant; } @@ -242,7 +262,7 @@ class FunctionGlobal { } if (isConstant()) { - mConstant->_visitReferencedTypes(visitor); + visitor(mConstant); return; } @@ -280,20 +300,13 @@ class FunctionGlobal { } if (isConstant()) { - mConstant->_visitCompilerVisibleInternals(visitor); + visitor.visitTopo(mConstant); return; } if (isGlobalInDict() || isGlobalInCell() || isNamedModuleMember()) { PyObject* obj = extractGlobalRefFromDictOrCell(); - if (isNamedModuleMember() && mModuleName == "typed_python.compiler.native_compiler.native_ast" && mName == "Expression") { - std::cout << "Visiting it! " << (obj ? "not empty":"empty") << "\n"; - if (!obj) { - asm("int3"); - } - } - if (obj) { _visitCompilerVisibleInternalsInPyobj(obj, visitor); } @@ -322,35 +335,6 @@ class FunctionGlobal { } } - FunctionGlobal withConstantsInternalized(PyObjSnapshotMaker& maker) { - if (!(isGlobalInDict() || isGlobalInCell())) { - return *this; - } - - Type* t = getValueAsType(); - - if (t) { - auto it = maker.getGroupMap().find(t); - if (it != maker.getGroupMap().end()) { - return FunctionGlobal::Constant( - PyObjSnapshot::ForType(it->second) - ); - } else { - return FunctionGlobal::Constant( - PyObjSnapshot::ForType(t) - ); - } - } - - PyObject* value = getValueAsPyobj(); - - if (!value) { - return FunctionGlobal::Unbound(); - } - - return FunctionGlobal::Constant(maker.internalize(value)); - } - FunctionGlobal withUpdatedInternalTypePointers(const std::map& groupMap) { Type* t = getValueAsType(); @@ -363,7 +347,7 @@ class FunctionGlobal { if (it != groupMap.end()) { // we're actually a constant! return FunctionGlobal::Constant( - PyObjSnapshot::ForType(it->second) + it->second ); } @@ -451,7 +435,7 @@ class FunctionGlobal { buffer.writeStringObject(3, mModuleName); } else if (mKind == GlobalType::Constant) { - mConstant->serialize(context, buffer, 4); + context.serializeNativeType(mConstant, buffer, 4); } else if (mKind == GlobalType::GlobalInDict) { context.serializePythonObject(mModuleDictOrCell, buffer, 1); @@ -469,7 +453,7 @@ class FunctionGlobal { uint64_t kind = 0; std::string name, moduleName; PyObjectHolder cellOrModule; - PyObjSnapshot* constant = nullptr; + Type* constant; buffer.consumeCompoundMessage(wireType, [&](size_t fieldNumber, size_t wireType) { if (fieldNumber == 0) { @@ -485,7 +469,7 @@ class FunctionGlobal { moduleName = buffer.readStringObject(); } else if (fieldNumber == 4) { - constant = PyObjSnapshot::deserialize(context, buffer, wireType); + constant = context.deserializeNativeType(buffer, wireType); } }); @@ -555,7 +539,7 @@ class FunctionGlobal { private: GlobalType mKind; - PyObjSnapshot* mConstant; + Type* mConstant; std::string mName; std::string mModuleName; diff --git a/typed_python/FunctionOverload.hpp b/typed_python/FunctionOverload.hpp index 5eb20dc74..a71942d41 100644 --- a/typed_python/FunctionOverload.hpp +++ b/typed_python/FunctionOverload.hpp @@ -288,9 +288,9 @@ class FunctionOverload { } } - void internalizeConstants(PyObjSnapshotMaker& snapMaker) { + void internalizeConstants(const std::map& typeMap) { for (auto& nameAndGlobal: mGlobals) { - nameAndGlobal.second = nameAndGlobal.second.withConstantsInternalized(snapMaker); + nameAndGlobal.second = nameAndGlobal.second.withConstantsInternalized(typeMap); } } diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index d529b613f..dd47ddfcf 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -160,9 +160,9 @@ class Function : public Type { // replace any direct references to PyObject we're holding internally // with PyObjSnapshot references instead - void internalizeConstants(PyObjSnapshotMaker& snapMaker) { + void internalizeConstants(const std::map& typeMap) { for (auto& o: mOverloads) { - o.internalizeConstants(snapMaker); + o.internalizeConstants(typeMap); } } diff --git a/typed_python/PyFunctionGlobal.cpp b/typed_python/PyFunctionGlobal.cpp index 37a0afd2e..641b1a406 100644 --- a/typed_python/PyFunctionGlobal.cpp +++ b/typed_python/PyFunctionGlobal.cpp @@ -160,7 +160,7 @@ PyObject* PyFunctionGlobal::tp_getattro(PyObject* selfObj, PyObject* attrName) { } if (attr == "constant" && global.isConstant()) { - return PyPyObjSnapshot::newPyObjSnapshot(global.getConstant()); + return incref((PyObject*)PyInstance::typeObj(global.getConstant())); } if (attr == "name" && (global.isNamedModuleMember() || global.isGlobalInDict())) { diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index d01e13cd6..78b3f1d59 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -476,21 +476,9 @@ void Type::attemptToResolve() { // allow each Function type to build CompilerVisiblePythonObjects for its globals // after this, any FunctionGlobals will refer to Constants not cells - std::unordered_map compilerVisiblePyObjMap; - std::unordered_map compilerVisibleTypeMap; - std::unordered_map compilerVisibleInstanceMap; - PyObjSnapshotMaker snapMaker( - compilerVisiblePyObjMap, - compilerVisibleTypeMap, - compilerVisibleInstanceMap, - resolutionMapping, - nullptr, - true - ); - for (auto typeAndSource: resolutionSource) { if (typeAndSource.first->isFunction()) { - ((Function*)typeAndSource.first)->internalizeConstants(snapMaker); + ((Function*)typeAndSource.first)->internalizeConstants(resolutionMapping); } } diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index ccaa4b80d..77eb2be54 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -173,7 +173,7 @@ def test_type_looks_resolvable_alternative(): bGlobal = A.f.overloads[0].globals['B'] assert bGlobal.kind == "Constant" - assert bGlobal.constant.kind == "Type" + assert bGlobal.constant.kind == "PrimitiveType" assert bGlobal.constant.type is B bFwdGlobal = AFwd[0].f.overloads[0].globals['B'] From a74f3d61de5473036437ed2eb236f18546b12d0a Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 31 Jul 2023 22:13:38 +0000 Subject: [PATCH 73/83] PyObjSnapshotMaker in its own file, no more 'groupMap' crap. --- typed_python/PyObjSnapshot.cpp | 288 +--------------------------- typed_python/PyObjSnapshot.hpp | 67 +------ typed_python/PyObjSnapshotMaker.cpp | 280 +++++++++++++++++++++++++++ typed_python/PyObjSnapshotMaker.hpp | 86 +++++++++ typed_python/PyPyObjSnapshot.cpp | 2 - typed_python/all.cpp | 1 + 6 files changed, 370 insertions(+), 354 deletions(-) create mode 100644 typed_python/PyObjSnapshotMaker.cpp create mode 100644 typed_python/PyObjSnapshotMaker.hpp diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 19ad4211d..1c22b074b 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -1,4 +1,5 @@ #include "PyObjSnapshot.hpp" +#include "PyObjSnapshotMaker.hpp" #include "PyObjGraphSnapshot.hpp" #include "PyObjRehydrator.hpp" @@ -207,293 +208,6 @@ std::string PyObjSnapshot::kindAsString() const { } -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionArg& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBinding& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBindingStep& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const MemberDefinition& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionGlobal& def) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeInternalizedOf(def, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(inMethods, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(inMethods, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(inMethods, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(inMethods, *this); - - return res; -} - -PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& inMethods) { - PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } - - res->becomeBundleOf(inMethods, *this); - - return res; -} - -/* static */ -PyObjSnapshot* PyObjSnapshotMaker::internalize(PyObject* val) -{ - Type* t = PyInstance::extractTypeFrom(val, true); - if (t) { - return internalize(t); - } - - ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); - if (instanceType) { - return internalize( - InstanceRef( - ((PyInstance*)val)->dataPtr(), - instanceType - ) - ); - } - - auto it = mObjMapCache.find(val); - - if (it != mObjMapCache.end()) { - return it->second; - } - - mObjMapCache[val] = new PyObjSnapshot(mGraph); - - if (mGraph) { - mGraph->registerSnapshot(mObjMapCache[val]); - } - - mObjMapCache[val]->becomeInternalizedOf(val, *this); - - return mObjMapCache[val]; -} - -/* static */ -PyObjSnapshot* PyObjSnapshotMaker::internalize(Type* val) -{ - // TODO: this doesn't really belong here. This infra is more general. - if (mGroupMap.find(val) != mGroupMap.end()) { - val = mGroupMap.find(val)->second; - } else { - if (val->isForwardDefined()) { - if (val->isResolved()) { - val = val->forwardResolvesTo(); - } - } - } - - auto it = mTypeMapCache.find(val); - - if (it != mTypeMapCache.end()) { - return it->second; - } - - mTypeMapCache[val] = new PyObjSnapshot(mGraph); - - if (mGraph) { - mGraph->registerSnapshot(mTypeMapCache[val]); - } - - mTypeMapCache[val]->becomeInternalizedOf(val, *this); - - return mTypeMapCache[val]; -} - -/* static */ -PyObjSnapshot* PyObjSnapshotMaker::internalize(InstanceRef val) -{ - Type* t = val.extractType(true); - if (t) { - return internalize(t); - } - - PyObject* o = val.extractPyobj(); - if (o) { - return internalize(o); - } - - auto it = mInstanceCache.find(val); - - if (it != mInstanceCache.end()) { - return it->second; - } - - mInstanceCache[val] = new PyObjSnapshot(mGraph); - - if (mGraph) { - mGraph->registerSnapshot(mInstanceCache[val]); - } - - mInstanceCache[val]->becomeInternalizedOf(val, *this); - - return mInstanceCache[val]; -} void PyObjSnapshot::becomeInternalizedOf( diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index adbc88fd4..514567ed0 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -20,12 +20,14 @@ #include "Type.hpp" #include "Instance.hpp" #include "PythonTypeInternals.hpp" +#include "PyObjSnapshotMaker.hpp" class PyObjGraphSnapshot; class PyObjSnapshot; class FunctionGlobal; class FunctionOverload; class PyObjRehydrator; +class PyObjSnapshotMaker; /********************************* PyObjSnapshot @@ -43,71 +45,6 @@ gives us a self-consistent view of the world to compile against so we don't have changing underneath us. **********************************/ -class PyObjSnapshotMaker { -public: - PyObjSnapshotMaker( - std::unordered_map& inObjMapCache, - std::unordered_map& inTypeMapCache, - std::unordered_map& inInstanceCache, - const std::map<::Type*, ::Type*>& inGroupMap, - PyObjGraphSnapshot* inGraph, - bool inLinkBackToOriginalObject - ) : - mObjMapCache(inObjMapCache), - mTypeMapCache(inTypeMapCache), - mInstanceCache(inInstanceCache), - mGroupMap(inGroupMap), - mGraph(inGraph), - mLinkBackToOriginalObject(inLinkBackToOriginalObject) - { - } - - PyObjSnapshot* internalize(const std::string& def); - PyObjSnapshot* internalize(const MemberDefinition& def); - PyObjSnapshot* internalize(const FunctionGlobal& def); - PyObjSnapshot* internalize(const FunctionOverload& def); - PyObjSnapshot* internalize(const FunctionArg& def); - PyObjSnapshot* internalize(const ClosureVariableBinding& def); - PyObjSnapshot* internalize(const ClosureVariableBindingStep& def); - PyObjSnapshot* internalize(const std::vector& def); - PyObjSnapshot* internalize(const std::vector& inArgs); - PyObjSnapshot* internalize(const std::vector& inArgs); - PyObjSnapshot* internalize(const std::vector& inArgs); - PyObjSnapshot* internalize(const std::vector& inArgs); - PyObjSnapshot* internalize(const std::map& inBindings); - PyObjSnapshot* internalize(const std::map& inGlobals); - PyObjSnapshot* internalize(const std::map& inMethods); - PyObjSnapshot* internalize(const std::map& inMethods); - PyObjSnapshot* internalize(const std::vector& inMethods); - PyObjSnapshot* internalize(PyObject* val); - PyObjSnapshot* internalize(Type* val); - PyObjSnapshot* internalize(const Instance& val) { - return internalize(val.ref()); - } - PyObjSnapshot* internalize(InstanceRef val); - - bool linkBackToOriginalObject() const { - return mLinkBackToOriginalObject; - } - - PyObjGraphSnapshot* graph() const { - return mGraph; - } - - const std::map<::Type*, ::Type*>& getGroupMap() const { - return mGroupMap; - } - -private: - std::unordered_map& mObjMapCache; - std::unordered_map& mTypeMapCache; - std::unordered_map& mInstanceCache; - const std::map<::Type*, ::Type*>& mGroupMap; - PyObjGraphSnapshot* mGraph; - bool mLinkBackToOriginalObject; -}; - - class PyObjSnapshot { private: friend class PyObjSnapshotMaker; diff --git a/typed_python/PyObjSnapshotMaker.cpp b/typed_python/PyObjSnapshotMaker.cpp new file mode 100644 index 000000000..db6ff1e00 --- /dev/null +++ b/typed_python/PyObjSnapshotMaker.cpp @@ -0,0 +1,280 @@ +#include "PyObjSnapshot.hpp" +#include "PyObjSnapshotMaker.hpp" + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionArg& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBinding& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBindingStep& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const MemberDefinition& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionGlobal& def) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeInternalizedOf(def, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& inMethods) { + PyObjSnapshot* res = new PyObjSnapshot(mGraph); + if (mGraph) { + mGraph->registerSnapshot(res); + } + + res->becomeBundleOf(inMethods, *this); + + return res; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(PyObject* val) +{ + Type* t = PyInstance::extractTypeFrom(val, true); + if (t) { + return internalize(t); + } + + ::Type* instanceType = PyInstance::extractTypeFrom(val->ob_type); + if (instanceType) { + return internalize( + InstanceRef( + ((PyInstance*)val)->dataPtr(), + instanceType + ) + ); + } + + auto it = mObjMapCache.find(val); + + if (it != mObjMapCache.end()) { + return it->second; + } + + mObjMapCache[val] = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(mObjMapCache[val]); + } + + mObjMapCache[val]->becomeInternalizedOf(val, *this); + + return mObjMapCache[val]; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(Type* val) +{ + auto it = mTypeMapCache.find(val); + + if (it != mTypeMapCache.end()) { + return it->second; + } + + mTypeMapCache[val] = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(mTypeMapCache[val]); + } + + mTypeMapCache[val]->becomeInternalizedOf(val, *this); + + return mTypeMapCache[val]; +} + +/* static */ +PyObjSnapshot* PyObjSnapshotMaker::internalize(InstanceRef val) +{ + Type* t = val.extractType(true); + if (t) { + return internalize(t); + } + + PyObject* o = val.extractPyobj(); + if (o) { + return internalize(o); + } + + auto it = mInstanceCache.find(val); + + if (it != mInstanceCache.end()) { + return it->second; + } + + mInstanceCache[val] = new PyObjSnapshot(mGraph); + + if (mGraph) { + mGraph->registerSnapshot(mInstanceCache[val]); + } + + mInstanceCache[val]->becomeInternalizedOf(val, *this); + + return mInstanceCache[val]; +} diff --git a/typed_python/PyObjSnapshotMaker.hpp b/typed_python/PyObjSnapshotMaker.hpp new file mode 100644 index 000000000..3628ceedf --- /dev/null +++ b/typed_python/PyObjSnapshotMaker.hpp @@ -0,0 +1,86 @@ +/****************************************************************************** + Copyright 2017-2023 typed_python Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +#pragma once + +#include +#include "Type.hpp" +#include "Instance.hpp" +#include "PythonTypeInternals.hpp" + +class PyObjGraphSnapshot; +class PyObjSnapshot; +class FunctionGlobal; +class FunctionOverload; + + +class PyObjSnapshotMaker { +public: + PyObjSnapshotMaker( + std::unordered_map& inObjMapCache, + std::unordered_map& inTypeMapCache, + std::unordered_map& inInstanceCache, + PyObjGraphSnapshot* inGraph, + bool inLinkBackToOriginalObject + ) : + mObjMapCache(inObjMapCache), + mTypeMapCache(inTypeMapCache), + mInstanceCache(inInstanceCache), + mGraph(inGraph), + mLinkBackToOriginalObject(inLinkBackToOriginalObject) + { + } + + PyObjSnapshot* internalize(const std::string& def); + PyObjSnapshot* internalize(const MemberDefinition& def); + PyObjSnapshot* internalize(const FunctionGlobal& def); + PyObjSnapshot* internalize(const FunctionOverload& def); + PyObjSnapshot* internalize(const FunctionArg& def); + PyObjSnapshot* internalize(const ClosureVariableBinding& def); + PyObjSnapshot* internalize(const ClosureVariableBindingStep& def); + PyObjSnapshot* internalize(const std::vector& def); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::vector& inArgs); + PyObjSnapshot* internalize(const std::map& inBindings); + PyObjSnapshot* internalize(const std::map& inGlobals); + PyObjSnapshot* internalize(const std::map& inMethods); + PyObjSnapshot* internalize(const std::map& inMethods); + PyObjSnapshot* internalize(const std::vector& inMethods); + PyObjSnapshot* internalize(PyObject* val); + PyObjSnapshot* internalize(Type* val); + PyObjSnapshot* internalize(const Instance& val) { + return internalize(val.ref()); + } + PyObjSnapshot* internalize(InstanceRef val); + + bool linkBackToOriginalObject() const { + return mLinkBackToOriginalObject; + } + + PyObjGraphSnapshot* graph() const { + return mGraph; + } + +private: + std::unordered_map& mObjMapCache; + std::unordered_map& mTypeMapCache; + std::unordered_map& mInstanceCache; + PyObjGraphSnapshot* mGraph; + bool mLinkBackToOriginalObject; +}; + diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index c71e247ac..024a4e6dd 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -42,7 +42,6 @@ PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwar std::unordered_map constantMapCache; std::unordered_map constantMapCacheType; std::unordered_map constantMapCacheInst; - std::map<::Type*, ::Type*> groupMap; PyObjGraphSnapshot* graph = new PyObjGraphSnapshot(); @@ -50,7 +49,6 @@ PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwar constantMapCache, constantMapCacheType, constantMapCacheInst, - groupMap, graph, linkBack ); diff --git a/typed_python/all.cpp b/typed_python/all.cpp index fa1c9c216..b639da242 100644 --- a/typed_python/all.cpp +++ b/typed_python/all.cpp @@ -79,6 +79,7 @@ compile the entire group all at once. #include "PyFunctionOverload.cpp" #include "PyFunctionGlobal.cpp" #include "PyObjSnapshot.cpp" +#include "PyObjSnapshotMaker.cpp" #include "PyObjGraphSnapshot.cpp" #include "PyObjRehydrator.cpp" #include "PyPyObjSnapshot.cpp" From cd91da4affe197279cbd909a2138e3e2ba65d80f Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 31 Jul 2023 22:21:30 +0000 Subject: [PATCH 74/83] [wip] --- typed_python/Type.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 78b3f1d59..6553fb982 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -255,6 +255,14 @@ bool Type::canConvertToTrivially(Type* otherType) { } void reachableUnresolvedTypes(Type* root, std::set& outTypes) { + PyObjGraphSnapshot graph; + std::unordered_map objMapCache; + std::unordered_map typeMapCache; + std::unordered_map instanceCache; + + + PyObjSnapshotMaker snapMaker(objMapCache, typeMapCache, instanceCache, &graph, true); + std::set toVisit; toVisit.insert(root); From d76ad202fdb9b3d196ace298700060d0a43cad54 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 1 Aug 2023 10:12:03 -0400 Subject: [PATCH 75/83] WIP --- typed_python/PyObjGraphSnapshot.cpp | 44 +++++++++++++++++++++++++++++ typed_python/PyObjGraphSnapshot.hpp | 39 +++++++++++++++++++++++++ typed_python/PyObjSnapshot.hpp | 18 +++++++++++- typed_python/Type.cpp | 7 ++++- 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index 4bb0de006..742bef0d7 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -233,3 +233,47 @@ PyObjSnapshot* PyObjGraphSnapshot::pickRootFor(const std::unordered_set& links) { + for (auto snap: mObjects) { + if ( + snap->getKind() == PyObjSnapshot::Kind::ForwardType && + snap->hasNamedElement("fwd_cell_or_dict") + ) { + links.insert( + FwdDefLink( + snap->getNamedElement("fwd_cell_or_dict")->getPyObj(), + snap->getName() + ) + ); + } else + if (snap->getKind() == PyObjSnapshot::Kind::FunctionGlobal) { + if (snap->hasNamedElement("moduleDict")) { + links.insert( + FwdDefLink( + snap->getNamedElement("moduleDict"), + snap->getName() + ) + ); + } else if (snap->hasNamedElement("cell")) { + links.insert( + FwdDefLink( + snap->getNamedElement("cell"), + "" + ) + ); + } + } + } +} + +void PyObjGraphSnapshot::getOutboundLinks(std::set& outTypes) { + for (auto snap: mObjects) { + if (snap->willBeATpType()) { + outTypes.insert(snap); + } + } + diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index 0985677c1..4f4a03ec5 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -37,6 +37,34 @@ A graph knows how to assign a unique hash to every object it contains. **********************************/ +// formally, a FwdDefLink is a forward definition of a python value +// contained in a cell or a module member whose definition may only +// change from being undefined or a forward to a concrete value. +class FwdDefLink { +public: + FwdDefLink(PyObject* cellOrDict, std::string name) : + mName(name), + mCellOrDict(cellOrDict) + { + } + + bool operator<(const FwdDefLink& in) const { + if (mCellOrDict < in.mCellOrDict) { + return true; + } + if (mCellOrDict > in.mCellOrDict) { + return false; + } + + return mName < in.mName; + } + +private: + PyObjectHolder mCellOrDict; + std::string mName; +}; + + class PyObjGraphSnapshot { public: PyObjGraphSnapshot() : @@ -72,6 +100,17 @@ class PyObjGraphSnapshot { ShaHash hashFor(PyObjSnapshot* snap); + // find every FunctionGlobal and Forward that's pointing at + // a cell or ModuleDict. These objects determine how our types + // actually get linked together into concrete definitions. + void getOutboundLinks(std::set& links); + + // all contained snapshots that represent TP types + void getTypes(std::set& outTypeSnaps); + + // + void getOutboundLinks(std::set& links); + private: void installSnapHash(PyObjSnapshot* snap, ShaHash h); diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 514567ed0..e286d6820 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -126,7 +126,9 @@ class PyObjSnapshot { ClassMemberDefinition, // a TP BoundMethod type BoundMethodType, - // a TP Forward type + // a TP Forward type. Contains fwd_target for the target, if set, + // fwd_cell_or_dict for a forward defined using a lambda, and + // mName if the forward dict was a global. ForwardType, // a TP TypedCellType TypedCellType, @@ -355,6 +357,20 @@ class PyObjSnapshot { return mNamedElements; } + bool hasNamedElement(std::string s) const { + return mNamedElements.find(s) != mNamedElements.end(); + } + + PyObjSnapshot* getNamedElement(std::string s) const { + auto it = mNamedElements.find(s); + + if (it != mNamedElements.end()) { + return it->second; + } + + return nullptr; + } + const std::vector& names() const { return mNames; } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 6553fb982..bb6254dbb 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -260,9 +260,14 @@ void reachableUnresolvedTypes(Type* root, std::set& outTypes) { std::unordered_map typeMapCache; std::unordered_map instanceCache; - PyObjSnapshotMaker snapMaker(objMapCache, typeMapCache, instanceCache, &graph, true); + // this finds every reachable python object from here ending at module objects. + // or fully resolved types. + + // if we have Forwards defined inside of us, then the target of the forward, if + // defined, will be contained in this graph. But if we have + std::set toVisit; toVisit.insert(root); From ce50cc04611b3eba0d6cf9ae878d873f3a558e77 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Tue, 1 Aug 2023 18:38:27 -0400 Subject: [PATCH 76/83] more --- todo.txt | 55 +++++++++ typed_python/FunctionGlobal.hpp | 2 +- typed_python/PyObjGraphSnapshot.cpp | 63 ++++------ typed_python/PyObjGraphSnapshot.hpp | 48 ++------ typed_python/PyObjSnapshot.cpp | 114 ++++++++---------- typed_python/PyPyObjGraphSnapshot.cpp | 24 ++++ typed_python/PyPyObjGraphSnapshot.hpp | 2 + typed_python/Type.cpp | 68 +++++++---- .../compiler/native_compiler/native_ast.py | 15 +-- 9 files changed, 221 insertions(+), 170 deletions(-) diff --git a/todo.txt b/todo.txt index ddcacea76..b33d2a489 100644 --- a/todo.txt +++ b/todo.txt @@ -208,3 +208,58 @@ execution plan: just like forward types, or to be passed into a Class definition 3. allow forward function types to be resolved just like regular function types * forward function instances also participate in instance resolution + + + + + +how to do this: +1. new graph needs a hashing methodology that's totally independent of MRTG +2. type internalization framework needs to use it + + + + + + + + + + + + + + + + + + + +D = Forward(lambda: X) + +class C: + MyD = D + + +class D(Class): + MyC = C + + +How should we handle this case? + +At some level, both C and D are 'forward defined' because 'C' has a forward reference to D and vice versa. + +So, now we will have a new C and a new D, triggered at the moment we constructed 'D'. + +Of course, if we allowed 'C' to escape then we have a problem + + + +Of course, when we define C we wont' have + +at the module level, we shouldn't be replacing 'C', we should be updating it? + +how can we tell the difference between these two? + +Should 'C' have been considered 'Forward Defined'? + diff --git a/typed_python/FunctionGlobal.hpp b/typed_python/FunctionGlobal.hpp index d6d99d32f..ecefef786 100644 --- a/typed_python/FunctionGlobal.hpp +++ b/typed_python/FunctionGlobal.hpp @@ -238,7 +238,7 @@ class FunctionGlobal { return mModuleDictOrCell; } - PyObject* extractGlobalRefFromDictOrCell() { + PyObject* extractGlobalRefFromDictOrCell() const { if (isGlobalInDict()) { return PyDict_GetItemString(mModuleDictOrCell, mName.c_str()); } diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index 742bef0d7..0931fd328 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -17,6 +17,29 @@ #include "PyObjGraphSnapshot.hpp" #include "PyObjSnapshotGroupSearch.hpp" +PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBak) : + mGroupsConsumed(0), + mGroupSearch(this) +{ + std::unordered_map objMapCache; + std::unordered_map typeMapCache; + std::unordered_map instanceCache; + + // this finds every reachable python objects from 'root', with the search terminating at + // named module objects and named module dicts. This is enough information to completely + // characterize a type's "identity hash", since there can be at most one version of a + // given named module per program. + PyObjSnapshotMaker snapMaker( + objMapCache, + typeMapCache, + instanceCache, + this, + linkBak + ); + + snapMaker.internalize(root); +} + template ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { if (snap->getKind() == PyObjSnapshot::Kind::Instance) { @@ -234,46 +257,10 @@ PyObjSnapshot* PyObjGraphSnapshot::pickRootFor(const std::unordered_set& links) { - for (auto snap: mObjects) { - if ( - snap->getKind() == PyObjSnapshot::Kind::ForwardType && - snap->hasNamedElement("fwd_cell_or_dict") - ) { - links.insert( - FwdDefLink( - snap->getNamedElement("fwd_cell_or_dict")->getPyObj(), - snap->getName() - ) - ); - } else - if (snap->getKind() == PyObjSnapshot::Kind::FunctionGlobal) { - if (snap->hasNamedElement("moduleDict")) { - links.insert( - FwdDefLink( - snap->getNamedElement("moduleDict"), - snap->getName() - ) - ); - } else if (snap->hasNamedElement("cell")) { - links.insert( - FwdDefLink( - snap->getNamedElement("cell"), - "" - ) - ); - } - } - } -} - -void PyObjGraphSnapshot::getOutboundLinks(std::set& outTypes) { +void PyObjGraphSnapshot::getTypes(std::unordered_set& outTypes) { for (auto snap: mObjects) { if (snap->willBeATpType()) { outTypes.insert(snap); } } - +} diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index 4f4a03ec5..e43fb63c4 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -37,34 +37,6 @@ A graph knows how to assign a unique hash to every object it contains. **********************************/ -// formally, a FwdDefLink is a forward definition of a python value -// contained in a cell or a module member whose definition may only -// change from being undefined or a forward to a concrete value. -class FwdDefLink { -public: - FwdDefLink(PyObject* cellOrDict, std::string name) : - mName(name), - mCellOrDict(cellOrDict) - { - } - - bool operator<(const FwdDefLink& in) const { - if (mCellOrDict < in.mCellOrDict) { - return true; - } - if (mCellOrDict > in.mCellOrDict) { - return false; - } - - return mName < in.mName; - } - -private: - PyObjectHolder mCellOrDict; - std::string mName; -}; - - class PyObjGraphSnapshot { public: PyObjGraphSnapshot() : @@ -73,12 +45,22 @@ class PyObjGraphSnapshot { { } + PyObjGraphSnapshot(Type* root, bool linkBackToOriginal=true); + ~PyObjGraphSnapshot() { for (auto oPtr: mObjects) { delete oPtr; } } + // walk over the graph and point any forwards to the type they actually point to. + // after this, any Kind::ForwardType will have a target set and empty name. + // Any outbound link pointing to a Forward will now point to what the forward was + // pointing to + void resolveForwards(); + + void internalize(); + // get the "internal" graph snapshot, which is responsible for holding all the objects // that are actually interned inside the system. static PyObjGraphSnapshot& internal() { @@ -100,16 +82,8 @@ class PyObjGraphSnapshot { ShaHash hashFor(PyObjSnapshot* snap); - // find every FunctionGlobal and Forward that's pointing at - // a cell or ModuleDict. These objects determine how our types - // actually get linked together into concrete definitions. - void getOutboundLinks(std::set& links); - // all contained snapshots that represent TP types - void getTypes(std::set& outTypeSnaps); - - // - void getOutboundLinks(std::set& links); + void getTypes(std::unordered_set& outTypeSnaps); private: void installSnapHash(PyObjSnapshot* snap, ShaHash h); diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 1c22b074b..e23d7fc82 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -15,194 +15,156 @@ std::string PyObjSnapshot::kindAsString() const { if (mKind == Kind::Uninitialized) { return "Uninitialized"; } - - if (mKind == Kind::InternalBundle) { - return "InternalBundle"; - } - if (mKind == Kind::String) { return "String"; } - if (mKind == Kind::NamedPyObject) { return "NamedPyObject"; } - - if (mKind == Kind::PrimitiveType) { - return "PrimitiveType"; - } - if (mKind == Kind::Instance) { return "Instance"; } - if (mKind == Kind::PrimitiveType) { return "PrimitiveType"; } - if (mKind == Kind::ListOfType) { return "ListOfType"; } - if (mKind == Kind::TupleOfType) { return "TupleOfType"; } - if (mKind == Kind::TupleType) { return "TupleType"; } - if (mKind == Kind::NamedTupleType) { return "NamedTupleType"; } - if (mKind == Kind::OneOfType) { return "OneOfType"; } - if (mKind == Kind::ValueType) { return "ValueType"; } - if (mKind == Kind::DictType) { return "DictType"; } - if (mKind == Kind::ConstDictType) { return "ConstDictType"; } - if (mKind == Kind::SetType) { return "SetType"; } - if (mKind == Kind::PointerToType) { return "PointerToType"; } - if (mKind == Kind::RefToType) { return "RefToType"; } - if (mKind == Kind::AlternativeType) { return "AlternativeType"; } - if (mKind == Kind::ConcreteAlternativeType) { return "ConcreteAlternativeType"; } - if (mKind == Kind::AlternativeMatcherType) { return "AlternativeMatcherType"; } - if (mKind == Kind::PythonObjectOfTypeType) { return "PythonObjectOfTypeType"; } - if (mKind == Kind::SubclassOfTypeType) { return "SubclassOfTypeType"; } - if (mKind == Kind::ClassType) { return "ClassType"; } - if (mKind == Kind::HeldClassType) { return "HeldClassType"; } - if (mKind == Kind::FunctionType) { return "FunctionType"; } - if (mKind == Kind::FunctionOverload) { return "FunctionOverload"; } - if (mKind == Kind::FunctionGlobal) { return "FunctionGlobal"; } - + if (mKind == Kind::FunctionArg) { + return "FunctionArg"; + } + if (mKind == Kind::FunctionClosureVariableBinding) { + return "FunctionClosureVariableBinding"; + } + if (mKind == Kind::FunctionClosureVariableBindingStep) { + return "FunctionClosureVariableBindingStep"; + } + if (mKind == Kind::ClassMemberDefinition) { + return "ClassMemberDefinition"; + } if (mKind == Kind::BoundMethodType) { return "BoundMethodType"; } - if (mKind == Kind::ForwardType) { return "ForwardType"; } - if (mKind == Kind::TypedCellType) { return "TypedCellType"; } - if (mKind == Kind::PyList) { return "PyList"; } - if (mKind == Kind::PyDict) { return "PyDict"; } - if (mKind == Kind::PySet) { return "PySet"; } - if (mKind == Kind::PyTuple) { return "PyTuple"; } - if (mKind == Kind::PyClass) { return "PyClass"; } - - if (mKind == Kind::PyFunction) { - return "PyFunction"; - } - if (mKind == Kind::PyObject) { return "PyObject"; } - - if (mKind == Kind::PyCell) { - return "PyCell"; + if (mKind == Kind::PyFunction) { + return "PyFunction"; } - if (mKind == Kind::PyCodeObject) { return "PyCodeObject"; } - + if (mKind == Kind::PyCell) { + return "PyCell"; + } if (mKind == Kind::PyModule) { return "PyModule"; } - if (mKind == Kind::PyModuleDict) { return "PyModuleDict"; } - if (mKind == Kind::PyClassDict) { return "PyClassDict"; } - if (mKind == Kind::PyStaticMethod) { return "PyStaticMethod"; } - if (mKind == Kind::PyClassMethod) { return "PyClassMethod"; } - if (mKind == Kind::PyBoundMethod) { return "PyBoundMethod"; } - if (mKind == Kind::PyProperty) { return "PyProperty"; } - if (mKind == Kind::ArbitraryPyObject) { return "ArbitraryPyObject"; } + if (mKind == Kind::InternalBundle) { + return "InternalBundle"; + } throw std::runtime_error("Unknown PyObjSnapshot Kind: " + format((int)mKind)); } @@ -428,11 +390,24 @@ void PyObjSnapshot::becomeInternalizedOf( mKind = Kind::ForwardType; Forward* f = (Forward*)t; - if (f->getTarget()) { - mNamedElements["fwd_target"] = maker.internalize(f->getTarget()); - } if (f->getCellOrDict()) { mNamedElements["fwd_cell_or_dict"] = maker.internalize(f->getCellOrDict()); + + // if we are a 'lambda' forward, we need to know what we resolve to, assuming + // we do resolve. + if (PyCell_Check(f->getCellOrDict()) && PyCell_Get(f->getCellOrDict())) { + mNamedElements["fwd_cell_resolves_to"] = maker.internalize(PyCell_Get(f->getCellOrDict())); + } else + if (PyDict_Check(f->getCellOrDict())) { + PyObject* o = PyDict_GetItemString(f->getCellOrDict(), f->name().c_str()); + if (o) { + mNamedElements["fwd_dict_resolves_to"] = maker.internalize(o); + } + } + } + + if (f->getTarget()) { + mNamedElements["fwd_target"] = maker.internalize(f->getTarget()); } mName = f->name(); @@ -868,6 +843,18 @@ void PyObjSnapshot::becomeInternalizedOf( mModuleName = val.getModuleName(); mName = val.getName(); mNamedElements["moduleDict"] = maker.internalize(val.getModuleDictOrCell()); + + // TODO: ultimately we should get rid of this. It shouldn't be necessary to + // establish a valid 'identity' and its a little arbitrary that we're only recursing + // into types instead of all python objects. For the moment, we need it to keep everything + // working. + PyObject* o = val.extractGlobalRefFromDictOrCell(); + if (o && PyType_Check(o) && PyInstance::extractTypeFrom(o, true)) { + mNamedElements["resolve_to"] = maker.internalize( + PyInstance::extractTypeFrom(o, true) + ); + } + return; } @@ -881,6 +868,11 @@ void PyObjSnapshot::becomeInternalizedOf( mName = val.getName(); return; } + + if (val.isConstant()) { + mNamedElements["constant"] = maker.internalize(val.getConstant()); + return; + } } void PyObjSnapshot::becomeInternalizedOf( diff --git a/typed_python/PyPyObjGraphSnapshot.cpp b/typed_python/PyPyObjGraphSnapshot.cpp index 5f49318f3..2bd26c047 100644 --- a/typed_python/PyPyObjGraphSnapshot.cpp +++ b/typed_python/PyPyObjGraphSnapshot.cpp @@ -22,6 +22,7 @@ PyDoc_STRVAR(PyPyObjGraphSnapshot_doc, ); PyMethodDef PyPyObjGraphSnapshot_methods[] = { + {"extractTypes", (PyCFunction)PyPyObjGraphSnapshot::extractTypes, METH_VARARGS | METH_KEYWORDS, NULL}, {NULL} /* Sentinel */ }; @@ -44,6 +45,29 @@ PyObject* PyPyObjGraphSnapshot::newPyObjGraphSnapshot(PyObjGraphSnapshot* g, boo return (PyObject*)self; } +/* static */ +PyObject* PyPyObjGraphSnapshot::extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + throw PythonExceptionSet(); + } + + PyObjectStealer res(PySet_New(NULL)); + + for (auto o: self->mGraphSnapshot->getObjects()) { + if (o->getType()) { + PySet_Add(res, (PyObject*)PyInstance::typeObj(o->getType())); + } + } + + return incref(res); + }); +} + /* static */ PyObject* PyPyObjGraphSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) { diff --git a/typed_python/PyPyObjGraphSnapshot.hpp b/typed_python/PyPyObjGraphSnapshot.hpp index 7dd773d25..5b1621219 100644 --- a/typed_python/PyPyObjGraphSnapshot.hpp +++ b/typed_python/PyPyObjGraphSnapshot.hpp @@ -30,6 +30,8 @@ class PyPyObjGraphSnapshot { static void dealloc(PyPyObjGraphSnapshot *self); + static PyObject* extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs); + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); static PyObject* newPyObjGraphSnapshot(PyObjGraphSnapshot* o, bool ownsIt); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index bb6254dbb..765a3f6b4 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -260,32 +260,33 @@ void reachableUnresolvedTypes(Type* root, std::set& outTypes) { std::unordered_map typeMapCache; std::unordered_map instanceCache; - PyObjSnapshotMaker snapMaker(objMapCache, typeMapCache, instanceCache, &graph, true); - - // this finds every reachable python object from here ending at module objects. - // or fully resolved types. - - // if we have Forwards defined inside of us, then the target of the forward, if - // defined, will be contained in this graph. But if we have - - std::set toVisit; - toVisit.insert(root); - - // walk the graph and determine all forwards that are not resolved - while (toVisit.size()) { - Type* toCheck = *toVisit.begin(); - toVisit.erase(toCheck); - - if (toCheck->isForwardDefined() && !toCheck->isResolved()) { - if (outTypes.find(toCheck) == outTypes.end()) { - outTypes.insert(toCheck); - - toCheck->visitReferencedTypes([&](Type* subtype) { - if (subtype->isForwardDefined() && !subtype->isResolved()) { - toVisit.insert(subtype); - } - }); - } + // this finds every reachable python objects from 'root', with the search terminating at + // named module objects and named module dicts. This is enough information to completely + // characterize a type's "identity hash", since there can be at most one version of a + // given named module per program. + PyObjSnapshotMaker snapMaker( + objMapCache, + typeMapCache, + instanceCache, + &graph, + true + ); + + snapMaker.internalize(root); + + std::unordered_set typeSnaps; + graph.getTypes(typeSnaps); + + for (auto snap: typeSnaps) { + Type* t = snap->getType(); + if (!t) { + throw std::runtime_error( + "Snap of kind " + snap->kindAsString() + " didn't produce a type" + ); + } + + if (t->isForwardDefined() && !t->isResolved()) { + outTypes.insert(t); } } } @@ -417,6 +418,21 @@ void Type::attemptToResolve() { // do the entire type resolution process while holding the GIL PyEnsureGilAcquired getTheGil; + // compute a snapshot starting with us as the root + // PyObjGraphSnapshot graph(this); + + // // take the graph and remove any forwards from it. We will now have a graph where + // // no object points directly to a forward anymore. Each forward will point to its + // // resolution, and every PyObjSnapshot has a link back to the original object or + // // type that it came from. + // graph.resolveForwards(); + + // // now take every non-forward node and make a version of it in the internal graph, + // // if it doesn't already exist. Then rehydrate it. + // graph.internalize(); + + // // now type in 'graph' can determine which fully realized type it maps to. + std::set typesNeedingResolution; reachableUnresolvedTypes(this, typesNeedingResolution); diff --git a/typed_python/compiler/native_compiler/native_ast.py b/typed_python/compiler/native_compiler/native_ast.py index cfd48ab45..5dcdf8d28 100644 --- a/typed_python/compiler/native_compiler/native_ast.py +++ b/typed_python/compiler/native_compiler/native_ast.py @@ -88,8 +88,8 @@ def typeZeroConstant(self): raise Exception(f"Can't make a zero value from {self}") -Type = Forward() -Type.define(Alternative( +Type = Forward(lambda: Type) +Type = Alternative( "Type", # the 'Void' type. should only be used in function signatures. Use an empty # struct if you want the 'empty' data structure instead. @@ -109,7 +109,7 @@ def typeZeroConstant(self): all(valType.isEmpty() for keyType, valType in self.element_types) if self.matches.Struct else False, zeroConstant=typeZeroConstant -)) +) def const_truth_value(c): @@ -145,8 +145,8 @@ def const_str(c): assert False, type(c) -Constant = Forward() -Constant.define(Alternative( +Constant = Forward(lambda: Constant) +Constant = Alternative( "Constant", Void={}, Float={'val': float, 'bits': int}, @@ -157,7 +157,8 @@ def const_str(c): NullPointer={'value_type': Type}, truth_value=const_truth_value, __str__=const_str -)) +) + UnaryOp = Alternative( "UnaryOp", @@ -661,11 +662,11 @@ def expr_could_throw(self): )) +Expression = resolveForwardDefinedType(Expression) NamedCallTarget = resolveForwardDefinedType(NamedCallTarget) Type = resolveForwardDefinedType(Type) Constant = resolveForwardDefinedType(Constant) ExpressionIntermediate = resolveForwardDefinedType(ExpressionIntermediate) -Expression = resolveForwardDefinedType(Expression) Teardown = resolveForwardDefinedType(Teardown) CallTarget = resolveForwardDefinedType(CallTarget) From 1eb0d0646df33d9dc5083141439a480d281eb9a7 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 2 Aug 2023 12:27:00 +0000 Subject: [PATCH 77/83] WIP --- typed_python/PyObjGraphSnapshot.cpp | 11 +++++++++++ typed_python/PyObjGraphSnapshot.hpp | 5 ++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index 0931fd328..b2702976b 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -40,6 +40,17 @@ PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBak) : snapMaker.internalize(root); } +PyObjGraphSnapshot::resolveForwards() { + std::set fwdDefined; + + for (auto o: mObjects) { + if (o->isForwardDefinedType()) { + fwdDefined.insert(o); + } + } +} + + template ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { if (snap->getKind() == PyObjSnapshot::Kind::Instance) { diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index e43fb63c4..0f76a2db1 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -56,11 +56,10 @@ class PyObjGraphSnapshot { // walk over the graph and point any forwards to the type they actually point to. // after this, any Kind::ForwardType will have a target set and empty name. // Any outbound link pointing to a Forward will now point to what the forward was - // pointing to + // pointing to. Any Snapshots that had valid caches that were modified or that can + // reach modified values will have their snapshots cleared. void resolveForwards(); - void internalize(); - // get the "internal" graph snapshot, which is responsible for holding all the objects // that are actually interned inside the system. static PyObjGraphSnapshot& internal() { From 95a804d2d2fb2fedb70c190e429e3adb3f6909ee Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Wed, 2 Aug 2023 22:30:35 +0000 Subject: [PATCH 78/83] WIP --- typed_python/PyObjGraphSnapshot.cpp | 109 +++++++++++++++++++++- typed_python/PyObjGraphSnapshot.hpp | 12 ++- typed_python/PyObjSnapshot.cpp | 66 +++++++++++++ typed_python/PyObjSnapshot.hpp | 139 ++++++++++++++++++---------- typed_python/Type.cpp | 23 ++--- typed_python/Type.hpp | 12 ++- 6 files changed, 296 insertions(+), 65 deletions(-) diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index b2702976b..c8931df5b 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -40,14 +40,106 @@ PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBak) : snapMaker.internalize(root); } -PyObjGraphSnapshot::resolveForwards() { - std::set fwdDefined; +/* walk over the graph and point any forwards to the type they actually point to. + +This will modify nodes in the following way: + Any Kind::ForwardType will have a target set if its resolvable + Any snapshot pointing at a Forward will now point at what the forward points to + Any snapshot with a valid cache that can reach a type that was changed will have its + cache cleared + +*/ +void PyObjGraphSnapshot::resolveForwards() { + // any snapshot that was modified + std::set modified; + + // point forwards where they go + for (auto o: mObjects) { + if (o->getKind() == PyObjSnapshot::Kind::ForwardType) { + o->pointForwardToFinalType(); + modified.insert(o); + } + + if (o->willBeATpType()) { + if (o->markTypeNotFwdDefined()) { + modified.insert(0); + } + } + } for (auto o: mObjects) { - if (o->isForwardDefinedType()) { - fwdDefined.insert(o); + if (o->replaceOutboundForwardsWithTargets()) { + modified.insert(o); } } + + // build a reverse graph map + std::map > inbound; + + for (auto o: mObjects) { + o->visitOutbound([&](PyObjSnapshot* outbound) { + if (outbound->getGraph() == this) { + inbound[outbound].insert(o); + } + }); + } + + std::set modifiedTransitive; + std::function check = [&](PyObjSnapshot* s) { + if (modifiedTransitive.find(s) == modifiedTransitive.end()) { + modifiedTransitive.insert(s); + for (auto upstream: inbound[s]) { + check(upstream); + } + } + }; + + for (auto m: modified) { + check(m); + } + + for (auto m: modifiedTransitive) { + m->clearCache(); + } +} + + +void PyObjGraphSnapshot::internalize() { + std::map > skeletons; + + for (auto s: mObjects) { + ShaHash h = hashFor(s); + + if (!internal().snapshotForHash(h)) { + skeletons[h] = std::make_pair(s, internal().createSkeleton(h)); + } + } + + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->cloneFromSnapByHash(hashAndSkeleton.second.first); + } + + // rehydrate this portion of the graph, so that we have concrete objects for everything + // we just interned + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->rehydrate(); + } + + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->markInternalizedOnType(); + } +} + +PyObjSnapshot* PyObjGraphSnapshot::createSkeleton(ShaHash h) { + if (mHashToSnap.find(h) != mHashToSnap.end()) { + throw std::runtime_error("Can't create a skeleton for an object that already exists"); + } + + PyObjSnapshot* snap = new PyObjSnapshot(this); + + mHashToSnap[h] = snap; + mObjects.insert(snap); + return snap; } @@ -127,6 +219,15 @@ ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { } +PyObjSnapshot* PyObjGraphSnapshot::snapshotForHash(ShaHash hash) { + auto it = mHashToSnap.find(hash); + if (it != mHashToSnap.end()) { + return it->second; + } + return nullptr; +} + + ShaHash PyObjGraphSnapshot::hashFor(PyObjSnapshot* snap) { if (snap->getGraph() != this) { throw std::runtime_error("Can't graph a hash of a PyObjSnapshot we don't have"); diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index 0f76a2db1..05505c4b9 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -56,10 +56,14 @@ class PyObjGraphSnapshot { // walk over the graph and point any forwards to the type they actually point to. // after this, any Kind::ForwardType will have a target set and empty name. // Any outbound link pointing to a Forward will now point to what the forward was - // pointing to. Any Snapshots that had valid caches that were modified or that can - // reach modified values will have their snapshots cleared. + // pointing to. Any Snapshots that had valid caches that were modified or that can + // reach modified values will have their snapshots cleared. void resolveForwards(); + // make a copy of every object in here in the internal graph, matched up by + // sha hash + void internalize(); + // get the "internal" graph snapshot, which is responsible for holding all the objects // that are actually interned inside the system. static PyObjGraphSnapshot& internal() { @@ -81,10 +85,14 @@ class PyObjGraphSnapshot { ShaHash hashFor(PyObjSnapshot* snap); + PyObjSnapshot* snapshotForHash(ShaHash h); + // all contained snapshots that represent TP types void getTypes(std::unordered_set& outTypeSnaps); private: + PyObjSnapshot* createSkeleton(ShaHash h); + void installSnapHash(PyObjSnapshot* snap, ShaHash h); void computeHashesFor(const std::unordered_set& group); diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index e23d7fc82..d9a92ea26 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -5,6 +5,10 @@ void PyObjSnapshot::rehydrate() { + if (!needsHydration()) { + return; + } + PyObjRehydrator rehydrator; rehydrator.start(this); @@ -170,7 +174,42 @@ std::string PyObjSnapshot::kindAsString() const { } +void PyObjSnapshot::cloneFromSnapByHash(PyObjSnapshot* snap) { + mKind = snap->mKind; + mType = snap->mType; + mInstance = snap->mInstance; + mPyObject = incref(snap->mPyObject); + mNamedInts = snap->mNamedInts; + mNames = snap->mNames; + mStringValue = snap->mStringValue; + mModuleName = snap->mModuleName; + mName = snap->mName; + mQualname = snap->mQualname; + + auto intern = [&](PyObjSnapshot* s) { + PyObjSnapshot* res = mGraph->snapshotForHash( + snap->mGraph->hashFor(snap) + ); + + if (!res) { + throw std::runtime_error("Somehow, when interning, a hash is missing"); + } + + return res; + }; + for (auto e: snap->mElements) { + mElements.push_back(intern(e)); + } + + for (auto e: snap->mKeys) { + mKeys.push_back(intern(e)); + } + + for (auto nameAndE: snap->mNamedElements) { + mNamedElements[nameAndE.first] = intern(nameAndE.second); + } +} void PyObjSnapshot::becomeInternalizedOf( Type* t, @@ -969,3 +1008,30 @@ void PyObjSnapshot::becomeInternalizedOf( mKind = Kind::String; mStringValue = val; } + +PyObjSnapshot* PyObjSnapshot::computeForwardTargetTransitive() { + PyObjSnapshot* source = this; + int steps = 0; + + while (true) { + steps += 1; + if (steps > mGraph->getObjects().size()) { + throw std::runtime_error("Forward cycle detected"); + } + + if (source->mKind != Kind::ForwardType) { + // TODO - what if we're not a type? + if (!source->willBeATpType()) { + throw std::runtime_error("Should we be converting this to a Value type?"); + } + + return this; + } + + source = source->computeForwardTarget(); + if (!source) { + return nullptr; + } + } +} + diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index e286d6820..5b3e9e670 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -48,6 +48,7 @@ changing underneath us. class PyObjSnapshot { private: friend class PyObjSnapshotMaker; + friend class PyObjGraphSnapshot; friend class PyObjRehydrator; PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr) : @@ -176,15 +177,6 @@ class PyObjSnapshot { decref(mPyObject); } - static PyObjSnapshot* ForType(Type* t) { - // TODO: this needs to go away. its a shim to handle the current - // code which is a little confused about these objects. - PyObjSnapshot* res = new PyObjSnapshot(); - res->mKind = Kind::PrimitiveType; - res->mType = t; - return res; - } - Kind getKind() const { return mKind; } @@ -395,43 +387,6 @@ class PyObjSnapshot { return mQualname; } - template - void _visitReferencedTypes(const visitor_type& v) { - if (mKind == Kind::PrimitiveType) { - v(mType); - return; - } - - if (mKind == Kind::Instance) { - // TODO: what to do here? - } - - if (mKind == Kind::PyTuple) { - // TODO: what to do here? - } - } - - template - void _visitCompilerVisibleInternals(const visitor_type& visitor) { - if (mKind == Kind::PrimitiveType) { - visitor.visitTopo(mType); - } - - if (mKind == Kind::Instance) { - // TODO: what to do here? - visitor.visitInstance(mInstance.type(), mInstance.data()); - } - - if (mKind == Kind::PyTuple) { - // TODO: what to do here? - throw std::runtime_error("TODO: PyObjSnapshot::_visitCompilerVisibleInternals PyTuple"); - } - - if (mKind == Kind::ArbitraryPyObject) { - visitor.visitTopo(mPyObject); - } - } - bool willBeATpType() const { return mKind == Kind::PrimitiveType || mKind == Kind::ListOfType @@ -459,7 +414,6 @@ class PyObjSnapshot { ; } - ::Type* getType() { if (mType) { return mType; @@ -660,6 +614,22 @@ class PyObjSnapshot { return result; } + void clearCache() { + if (mKind == Kind::ArbitraryPyObject + || mKind == Kind::Instance + || mKind == Kind::PrimitiveType + ) { + throw std::runtime_error("Can't clear the cache of a leaf node."); + } + + mType = nullptr; + mInstance = Instance(); + if (mPyObject) { + decref(mPyObject); + mPyObject = nullptr; + } + } + template void visitOutbound(const visitor_type& v) { for (auto e: mElements) { @@ -675,7 +645,82 @@ class PyObjSnapshot { } } + void pointForwardToFinalType() { + if (mKind != Kind::ForwardType) { + throw std::runtime_error("Makes no sense to call this on a non-forward"); + } + + PyObjSnapshot* target = computeForwardTargetTransitive();; + + if (!target) { + throw std::runtime_error("Forward doesn't resolve to a valid target"); + } + + mNamedElements["fwd_target"] = target; + } + + // replace any outbound links that are pointing to forwards with the forward target + // returns true if we modified the object + bool replaceOutboundForwardsWithTargets() { + bool updated = false; + + visitOutbound([&](PyObjSnapshot*& snap) { + if (snap->mKind == Kind::ForwardType) { + PyObjSnapshot* tgt = snap->getNamedElement("fwd_target"); + if (!tgt) { + throw std::runtime_error("Somehow a forward doesn't have a target"); + } + snap = tgt; + updated = true; + } + }); + + return updated; + } + + PyObjSnapshot* computeForwardTarget() { + if (mKind != Kind::ForwardType) { + return nullptr; + } + PyObjSnapshot* snap = getNamedElement("fwd_target"); + if (snap) { + return snap; + } + + snap = getNamedElement("fwd_cell_resolves_to"); + if (snap) { + return snap; + } + + snap = getNamedElement("fwd_dict_resolves_to"); + if (snap) { + return snap; + } + return nullptr; + } + + PyObjSnapshot* computeForwardTargetTransitive(); + private: + void markInternalizeOnType() { + if (!mType) { + return; + } + + mType->setSnapshot(this); + } + + bool markTypeNotFwdDefined() { + if (mNamedInts["type_is_forward"]) { + mNamedInts["type_is_forward"] = 0; + return true; + } + + return false; + } + + void cloneFromSnapByHash(PyObjSnapshot* snap); + // ensure we won't crash if we interact with this object. void validateAfterDeserialization() { if (mKind == Kind::PrimitiveType) { diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 765a3f6b4..17ba7d37d 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -419,17 +419,18 @@ void Type::attemptToResolve() { PyEnsureGilAcquired getTheGil; // compute a snapshot starting with us as the root - // PyObjGraphSnapshot graph(this); - - // // take the graph and remove any forwards from it. We will now have a graph where - // // no object points directly to a forward anymore. Each forward will point to its - // // resolution, and every PyObjSnapshot has a link back to the original object or - // // type that it came from. - // graph.resolveForwards(); - - // // now take every non-forward node and make a version of it in the internal graph, - // // if it doesn't already exist. Then rehydrate it. - // graph.internalize(); + PyObjGraphSnapshot graph(this); + + // take the graph and remove any forwards from it. We will now have a graph where + // no object points directly to a forward anymore. Each forward will point to its + // resolution, and every PyObjSnapshot has a link back to the original object or + // type that it came from. + graph.resolveForwards(); + + // now take every non-forward node and make a version of it in the internal graph, + // if it doesn't already exist. Then rehydrate it. + unresolved object, then we didn't need to update it and it'll be hydrated as normal + graph.internalize(); // // now type in 'graph' can determine which fully realized type it maps to. diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 56fa1041a..3d357c532 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -43,6 +43,7 @@ class SerializationBuffer; class DeserializationBuffer; +class PyObjSnapshot; class Type; class NoneType; @@ -953,9 +954,12 @@ class Type { m_is_forward_defined(false), m_forward_resolves_to(nullptr), m_is_redundant(false), - m_is_being_deserialized(false) + m_is_being_deserialized(false), + mSnapshot(nullptr) {} + PyObjSnapshot* mSnapshot; + TypeCategory m_typeCategory; // this class is actively being deserialized. @@ -1054,6 +1058,12 @@ class Type { static std::map mInternalizedIdentityHashToType; + void setSnapshot(PyObjSnapshot* snapshot); + + PyObjSnapshot* getSnapshot() const { + return mSnapshot; + } + public: void addForwardDefinition(Type* t) { if (!t->isForwardDefined()) { From 0f28b43611bbb35e726b45c859ea42b5109e7310 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Thu, 3 Aug 2023 23:28:55 +0000 Subject: [PATCH 79/83] is this kind of working now? --- typed_python/AlternativeType.hpp | 4 + typed_python/ConcreteAlternativeType.hpp | 8 + typed_python/ForwardType.hpp | 3 + typed_python/PyInstance.cpp | 94 +++--- typed_python/PyObjGraphSnapshot.cpp | 50 +++- typed_python/PyObjGraphSnapshot.hpp | 12 +- typed_python/PyObjRehydrator.cpp | 115 +++++++- typed_python/PyObjSnapshot.cpp | 36 ++- typed_python/PyObjSnapshot.hpp | 37 ++- typed_python/PyObjSnapshotMaker.cpp | 63 ---- typed_python/PyPyObjGraphSnapshot.cpp | 84 +++++- typed_python/PyPyObjGraphSnapshot.hpp | 6 + typed_python/PyPyObjSnapshot.cpp | 36 ++- typed_python/PyPyObjSnapshot.hpp | 1 + typed_python/ShaHash.hpp | 23 ++ typed_python/Type.cpp | 347 +++++++++++++---------- typed_python/Type.hpp | 14 +- typed_python/_types.cpp | 11 +- typed_python/py_obj_snapshot_test.py | 31 +- 19 files changed, 658 insertions(+), 317 deletions(-) diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index c30f9bd48..862d93240 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -132,6 +132,10 @@ class Alternative : public Type { m_methods = methods; m_subtypes = types; m_subtypes_concrete = subtypes_concrete; + + if (m_subtypes.size() != m_subtypes_concrete.size()) { + throw std::runtime_error("Corrupt Alternative: #types != #subtypes"); + } } void initializeFromConcrete(Type* forwardDefinitionOfSelf) { diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index dbffbbe47..0d59577a1 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -50,6 +50,14 @@ class ConcreteAlternative : public Type { } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { + if (m_which < 0 || m_which >= m_alternative->subtypes().size()) { + throw std::runtime_error( + "invalid alternative index: " + + format(m_which) + " not in [0," + + format(m_alternative->subtypes().size()) + ")" + ); + } + return m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; } diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 37801b7dd..03e7e2e01 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -92,6 +92,7 @@ class Forward : public Type { } if (target == this) { + asm("int3"); throw std::runtime_error("Forward can't be resolved to itself."); } @@ -178,6 +179,8 @@ class Forward : public Type { throw std::runtime_error("Forward types should never be explicity instantiated."); } + void postInitializeConcrete() {}; + private: Type* mTarget; diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 61c8a3abf..3a13ddd3b 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -320,54 +320,56 @@ PyObject* PyInstance::extractPythonObjectConcrete(Type* eltType, instance_ptr da // static PyObject* PyInstance::tp_new_type(PyTypeObject *subtype, PyObject *args, PyObject *kwds) { - Type::TypeCategory catToProduce = ((NativeTypeCategoryWrapper*)subtype)->mCategory; - - if (catToProduce != Type::TypeCategory::catNamedTuple && - catToProduce != Type::TypeCategory::catAlternative) { - if (kwds && PyDict_Size(kwds)) { - PyErr_Format(PyExc_TypeError, "Type %S does not accept keyword arguments", (PyObject*)subtype); - return NULL; + return translateExceptionToPyObject([&]() -> PyObject* { + Type::TypeCategory catToProduce = ((NativeTypeCategoryWrapper*)subtype)->mCategory; + + if (catToProduce != Type::TypeCategory::catNamedTuple && + catToProduce != Type::TypeCategory::catAlternative) { + if (kwds && PyDict_Size(kwds)) { + PyErr_Format(PyExc_TypeError, "Type %S does not accept keyword arguments", (PyObject*)subtype); + return nullptr; + } } - } - if (catToProduce == Type::TypeCategory::catListOf) { return MakeListOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catTupleOf) { return MakeTupleOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catPointerTo ) { return MakePointerToType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catRefTo ) { return MakeRefToType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catTuple ) { return MakeTupleType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catConstDict ) { return MakeConstDictType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catSet ) { return MakeSetType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catDict ) { return MakeDictType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catOneOf ) { return MakeOneOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catNamedTuple ) { return MakeNamedTupleType(nullptr, args, kwds); } - if (catToProduce == Type::TypeCategory::catBool ) { return MakeBoolType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt8 ) { return MakeInt8Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt16 ) { return MakeInt16Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt32 ) { return MakeInt32Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catInt64 ) { return MakeInt64Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catFloat32 ) { return MakeFloat32Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catFloat64 ) { return MakeFloat64Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt8 ) { return MakeUInt8Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt16 ) { return MakeUInt16Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt32 ) { return MakeUInt32Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catUInt64 ) { return MakeUInt64Type(nullptr, args); } - if (catToProduce == Type::TypeCategory::catString ) { return MakeStringType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catBytes ) { return MakeBytesType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catEmbeddedMessage ) { return MakeEmbeddedMessageType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catPyCell ) { return MakePyCellType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catTypedCell ) { return MakeTypedCellType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catNone ) { return MakeNoneType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catValue ) { return MakeValueType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catBoundMethod ) { return MakeBoundMethodType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catAlternativeMatcher ) { return MakeAlternativeMatcherType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catFunction ) { return MakeFunctionType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catClass ) { return MakeClassType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catAlternative ) { return MakeAlternativeType(nullptr, args, kwds); } - if (catToProduce == Type::TypeCategory::catSubclassOf ) { return MakeSubclassOfType(nullptr, args); } - if (catToProduce == Type::TypeCategory::catForward ) { return MakeForwardType(nullptr, args, kwds); } - - PyErr_Format(PyExc_TypeError, "unknown TypeCategory %S", (PyObject*)subtype); - return NULL; + if (catToProduce == Type::TypeCategory::catListOf) { return (PyObject*)MakeListOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catTupleOf) { return (PyObject*)MakeTupleOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catPointerTo ) { return (PyObject*)MakePointerToType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catRefTo ) { return (PyObject*)MakeRefToType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catTuple ) { return (PyObject*)MakeTupleType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catConstDict ) { return (PyObject*)MakeConstDictType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catSet ) { return (PyObject*)MakeSetType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catDict ) { return (PyObject*)MakeDictType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catOneOf ) { return (PyObject*)MakeOneOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catNamedTuple ) { return (PyObject*)MakeNamedTupleType(nullptr, args, kwds); } + if (catToProduce == Type::TypeCategory::catBool ) { return (PyObject*)MakeBoolType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt8 ) { return (PyObject*)MakeInt8Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt16 ) { return (PyObject*)MakeInt16Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt32 ) { return (PyObject*)MakeInt32Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catInt64 ) { return (PyObject*)MakeInt64Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catFloat32 ) { return (PyObject*)MakeFloat32Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catFloat64 ) { return (PyObject*)MakeFloat64Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt8 ) { return (PyObject*)MakeUInt8Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt16 ) { return (PyObject*)MakeUInt16Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt32 ) { return (PyObject*)MakeUInt32Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catUInt64 ) { return (PyObject*)MakeUInt64Type(nullptr, args); } + if (catToProduce == Type::TypeCategory::catString ) { return (PyObject*)MakeStringType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catBytes ) { return (PyObject*)MakeBytesType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catEmbeddedMessage ) { return (PyObject*)MakeEmbeddedMessageType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catPyCell ) { return (PyObject*)MakePyCellType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catTypedCell ) { return (PyObject*)MakeTypedCellType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catNone ) { return (PyObject*)MakeNoneType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catValue ) { return (PyObject*)MakeValueType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catBoundMethod ) { return (PyObject*)MakeBoundMethodType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catAlternativeMatcher ) { return (PyObject*)MakeAlternativeMatcherType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catFunction ) { return (PyObject*)MakeFunctionType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catClass ) { return (PyObject*)MakeClassType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catAlternative ) { return (PyObject*)MakeAlternativeType(nullptr, args, kwds); } + if (catToProduce == Type::TypeCategory::catSubclassOf ) { return (PyObject*)MakeSubclassOfType(nullptr, args); } + if (catToProduce == Type::TypeCategory::catForward ) { return (PyObject*)MakeForwardType(nullptr, args, kwds); } + + PyErr_Format(PyExc_TypeError, "unknown TypeCategory %S", (PyObject*)subtype); + return nullptr; + }); } // static diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index c8931df5b..6fed9f10f 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -62,7 +62,7 @@ void PyObjGraphSnapshot::resolveForwards() { if (o->willBeATpType()) { if (o->markTypeNotFwdDefined()) { - modified.insert(0); + modified.insert(o); } } } @@ -105,13 +105,17 @@ void PyObjGraphSnapshot::resolveForwards() { void PyObjGraphSnapshot::internalize() { + internal().internalize(*this, false); +} + +void PyObjGraphSnapshot::internalize(PyObjGraphSnapshot& otherGraph, bool markInternalized) { std::map > skeletons; - for (auto s: mObjects) { - ShaHash h = hashFor(s); + for (auto s: otherGraph.mObjects) { + ShaHash h = otherGraph.hashFor(s); - if (!internal().snapshotForHash(h)) { - skeletons[h] = std::make_pair(s, internal().createSkeleton(h)); + if (!snapshotForHash(h)) { + skeletons[h] = std::make_pair(s, createSkeleton(h)); } } @@ -119,14 +123,25 @@ void PyObjGraphSnapshot::internalize() { hashAndSkeleton.second.second->cloneFromSnapByHash(hashAndSkeleton.second.first); } - // rehydrate this portion of the graph, so that we have concrete objects for everything - // we just interned - for (auto hashAndSkeleton: skeletons) { - hashAndSkeleton.second.second->rehydrate(); - } + if (markInternalized) { + try { + // rehydrate this portion of the graph, so that we have concrete objects for everything + // we just interned + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->rehydrate(); + } - for (auto hashAndSkeleton: skeletons) { - hashAndSkeleton.second.second->markInternalizedOnType(); + for (auto hashAndSkeleton: skeletons) { + hashAndSkeleton.second.second->markInternalizedOnType(); + } + } catch(...) { + // undo this since the state is somehow corrupt + for (auto hashAndSkeleton: skeletons) { + mObjects.erase(hashAndSkeleton.second.second); + mHashToSnap.erase(hashAndSkeleton.first); + } + throw; + } } } @@ -147,9 +162,18 @@ template ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { if (snap->getKind() == PyObjSnapshot::Kind::Instance) { if (snap->getInstance().type()->isPOD()) { - return ShaHash(int(snap->getKind())) + return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + ShaHash::SHA1(snap->getInstance().data(), snap->getInstance().type()->bytecount()); } + if (snap->getInstance().type()->isBytes()) { + static BytesType* b = BytesType::Make(); + + return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + + ShaHash::SHA1( + b->eltPtr(snap->getInstance().data(), 0), + b->count(snap->getInstance().data()) + ); + } throw std::runtime_error( "Can't hash a PyObjSnapshot Instance of type " + snap->getInstance().type()->name() ); diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index 05505c4b9..c799101ee 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -60,10 +60,12 @@ class PyObjGraphSnapshot { // reach modified values will have their snapshots cleared. void resolveForwards(); - // make a copy of every object in here in the internal graph, matched up by - // sha hash + // copy this into the 'internal' graph. void internalize(); + // internalize a graph into ourselves + void internalize(PyObjGraphSnapshot& graph, bool markInternalized); + // get the "internal" graph snapshot, which is responsible for holding all the objects // that are actually interned inside the system. static PyObjGraphSnapshot& internal() { @@ -71,12 +73,14 @@ class PyObjGraphSnapshot { return *graph; } - void registerSnapshot(PyObjSnapshot* obj) { + size_t registerSnapshot(PyObjSnapshot* obj) { if (obj->getGraph() != this) { throw std::runtime_error("Can't register a snapshot for a different graph"); } - + size_t ix = mObjects.size(); mObjects.insert(obj); + + return ix; } const std::unordered_set& getObjects() const { diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index 9a964ee8e..85667ebd0 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -7,11 +7,13 @@ void PyObjRehydrator::start(PyObjSnapshot* snapshot) { return; } - if (!snap->needsHydration()) { - return; - } + if (!snap->isUncacheable()) { + if (!snap->needsHydration()) { + return; + } - mSnapshots.insert(snap); + mSnapshots.insert(snap); + } for (auto elt: snap->mElements) { visit(elt); @@ -128,9 +130,9 @@ void PyObjRehydrator::getFrom( getNamedBundle(snapshot, "func_args", args); out = FunctionOverload( - getNamedElementPyobj(snapshot, "func_globals"), getNamedElementPyobj(snapshot, "func_code"), getNamedElementPyobj(snapshot, "func_defaults", true), + getNamedElementPyobj(snapshot, "func_annotations", true), globals, pyFuncClosureVarnames, globalsInClosureVarnames, @@ -261,6 +263,18 @@ PyObject* PyObjRehydrator::getNamedElementPyobj( ); } + if (!it->second) { + throw std::runtime_error("Corrupt PyObjSnapshot: nullptr in mNamedElements"); + } + + PyObject* o = pyobjFor(it->second); + + if (!o && !allowEmpty) { + throw std::runtime_error("Corrupt " + snapshot->toString() + + ": no pyobj for elt " + name + " which is " + it->second->toString() + ); + } + return pyobjFor(it->second); } @@ -378,6 +392,20 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { return; } + if (snap->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + snap->mType = new PythonObjectOfType(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + + PyObject* obType = getNamedElementPyobj(snap, "element_type", false); + + if (!PyType_Check(obType)) { + throw std::runtime_error("Can't rehydrate a PythonObjectOfType without a subclass of type"); + }; + + ((PythonObjectOfType*)snap->mType)->initializeDuringDeserialization((PyTypeObject*)obType); + return; + } + if (snap->mKind == PyObjSnapshot::Kind::TupleType) { snap->mType = new Tuple(); snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); @@ -870,6 +898,76 @@ void PyObjRehydrator::finalize() { for (auto n: mSnapshots) { finalizeRehydration(n); } + + std::vector types; + for (auto o: mSnapshots) { + if (o->mType) { + types.push_back(o->mType); + } + } + + std::sort(types.begin(), types.end(), [&](Type* l, Type* r) { + if (l->typeLevel() < r->typeLevel()) { + return true; + } + if (l->typeLevel() > r->typeLevel()) { + return false; + } + return l < r; + }); + + for (auto t: types) { + t->recomputeName(); + } + + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto t: types) { + if (t->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > types.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } + } + + // let each type update any internal caches it might need before it gets instantiated + for (auto t: types) { + t->finalizeType(); + } + + for (auto t: types) { + t->typeFinishedBeingDeserializedPhase1(); + } + + // now that we've set function globals, we need to go over all the classes and + // held classes and make sure the versions of the function types they're actually + // using have the new definitions. Unfortunately, Overload objects are not pointers + // (maybe they should be?) and so when we merge Overloads from base/child classes + // they don't get their globals replaced with the appropriate values + for (auto& t: types) { + if (t && t->isHeldClass()) { + ((HeldClass*)t)->mergeOwnFunctionsIntoInheritanceTree(); + } + } + + for (auto t: types) { + t->typeFinishedBeingDeserializedPhase2(); + } + + for (auto obj: mSnapshots) { + if (obj->mType) { + obj->mPyObject = incref((PyObject*)PyInstance::typeObj(obj->mType)); + } + } } void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { @@ -909,12 +1007,5 @@ void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { it->second->getPyObj() ); } - } else - if (obj->mType) { - obj->mType->recomputeName(); - obj->mType->finalizeType(); - obj->mType->typeFinishedBeingDeserializedPhase1(); - obj->mType->typeFinishedBeingDeserializedPhase2(); - obj->mPyObject = incref((PyObject*)PyInstance::typeObj(obj->mType)); } } diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index d9a92ea26..e4eb26c62 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -4,6 +4,19 @@ #include "PyObjRehydrator.hpp" + +PyObjSnapshot::PyObjSnapshot(PyObjGraphSnapshot* inGraph) : + mKind(Kind::Uninitialized), + mType(nullptr), + mPyObject(nullptr), + mGraph(inGraph), + mIndexInGraph(0) +{ + if (mGraph) { + mIndexInGraph = mGraph->registerSnapshot(this); + } +} + void PyObjSnapshot::rehydrate() { if (!needsHydration()) { return; @@ -188,7 +201,7 @@ void PyObjSnapshot::cloneFromSnapByHash(PyObjSnapshot* snap) { auto intern = [&](PyObjSnapshot* s) { PyObjSnapshot* res = mGraph->snapshotForHash( - snap->mGraph->hashFor(snap) + s->mGraph->hashFor(s) ); if (!res) { @@ -483,7 +496,7 @@ void PyObjSnapshot::becomeInternalizedOf( mPyObject = incref(val); } - PyObject* environType = staticPythonInstance("os", "_Environ"); + static PyObject* environType = staticPythonInstance("os", "_Environ"); if (val->ob_type == (PyTypeObject*)environType) { mKind = Kind::NamedPyObject; @@ -492,6 +505,23 @@ void PyObjSnapshot::becomeInternalizedOf( return; } + static PyObject* pyObj_object = staticPythonInstance("builtins", "object"); + + if (val == pyObj_object) { + mKind = Kind::NamedPyObject; + mName = "object"; + mModuleName = "builtins"; + return; + } + + static PyObject* pyObj_type = staticPythonInstance("builtins", "type"); + + if (val == pyObj_type) { + mKind = Kind::NamedPyObject; + mName = "type"; + mModuleName = "builtins"; + return; + } if (PyUnicode_Check(val)) { mKind = Kind::String; @@ -1025,7 +1055,7 @@ PyObjSnapshot* PyObjSnapshot::computeForwardTargetTransitive() { throw std::runtime_error("Should we be converting this to a Value type?"); } - return this; + return source; } source = source->computeForwardTarget(); diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 5b3e9e670..91e187588 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -51,13 +51,7 @@ class PyObjSnapshot { friend class PyObjGraphSnapshot; friend class PyObjRehydrator; - PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr) : - mKind(Kind::Uninitialized), - mType(nullptr), - mPyObject(nullptr), - mGraph(inGraph) - { - } + PyObjSnapshot(PyObjGraphSnapshot* inGraph=nullptr); public: enum class Kind { @@ -446,7 +440,15 @@ class PyObjSnapshot { return false; } - if (mKind == Kind::Uninitialized || mKind == Kind::InternalBundle) { + if (mKind == Kind::Uninitialized + || mKind == Kind::InternalBundle + || mKind == Kind::FunctionOverload + || mKind == Kind::FunctionGlobal + || mKind == Kind::FunctionArg + || mKind == Kind::FunctionClosureVariableBinding + || mKind == Kind::FunctionClosureVariableBindingStep + || mKind == Kind::ClassMemberDefinition + ) { return false; } @@ -457,6 +459,18 @@ class PyObjSnapshot { return true; } + // are we an object that doesn't actually produce a cacheable value + bool isUncacheable() const { + return mKind == Kind::InternalBundle + || mKind == Kind::FunctionOverload + || mKind == Kind::FunctionGlobal + || mKind == Kind::FunctionArg + || mKind == Kind::FunctionClosureVariableBinding + || mKind == Kind::FunctionClosureVariableBindingStep + || mKind == Kind::ClassMemberDefinition + ; + } + // get the python object representation of this object, which isn't guaranteed // to exist and may need to be constructed on demand. this will do a pass over // all reachable objects, building skeletons, and then performs a second pass where @@ -488,7 +502,7 @@ class PyObjSnapshot { void rehydrate(); std::string toString() const { - return "PyObjSnapshot." + kindAsString() + "()"; + return "PyObjSnapshot." + kindAsString() + "(ix=" + format(mIndexInGraph) + ")"; } template @@ -650,7 +664,7 @@ class PyObjSnapshot { throw std::runtime_error("Makes no sense to call this on a non-forward"); } - PyObjSnapshot* target = computeForwardTargetTransitive();; + PyObjSnapshot* target = computeForwardTargetTransitive(); if (!target) { throw std::runtime_error("Forward doesn't resolve to a valid target"); @@ -702,7 +716,7 @@ class PyObjSnapshot { PyObjSnapshot* computeForwardTargetTransitive(); private: - void markInternalizeOnType() { + void markInternalizedOnType() { if (!mType) { return; } @@ -738,6 +752,7 @@ class PyObjSnapshot { Kind mKind; PyObjGraphSnapshot* mGraph; + size_t mIndexInGraph; // if we are a PrimitiveType, then our type. If we resolve to a Type, a representation // of us as that type. Otherwise nullptr. diff --git a/typed_python/PyObjSnapshotMaker.cpp b/typed_python/PyObjSnapshotMaker.cpp index db6ff1e00..f48d11b16 100644 --- a/typed_python/PyObjSnapshotMaker.cpp +++ b/typed_python/PyObjSnapshotMaker.cpp @@ -5,9 +5,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -17,9 +14,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionArg& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -29,9 +23,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionArg& def) { PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBinding& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -41,9 +32,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBinding& def PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBindingStep& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -53,9 +41,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const ClosureVariableBindingStep& PyObjSnapshot* PyObjSnapshotMaker::internalize(const MemberDefinition& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -64,9 +49,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const MemberDefinition& def) { PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -75,9 +57,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionOverload& def) { PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(def, *this); @@ -86,9 +65,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(def, *this); @@ -97,9 +73,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& de PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(def, *this); @@ -108,9 +81,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& d PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(def, *this); @@ -119,9 +89,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& d PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vector& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(def, *this); @@ -130,9 +97,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::vectorregisterSnapshot(res); - } res->becomeInternalizedOf(def, *this); @@ -141,9 +105,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const FunctionGlobal& def) { PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(inMethods, *this); @@ -152,9 +113,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(inMethods, *this); @@ -163,9 +121,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(inMethods, *this); @@ -174,9 +129,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(inMethods, *this); @@ -185,9 +137,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::map& inMethods) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(res); - } res->becomeBundleOf(inMethods, *this); @@ -220,10 +169,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(PyObject* val) mObjMapCache[val] = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(mObjMapCache[val]); - } - mObjMapCache[val]->becomeInternalizedOf(val, *this); return mObjMapCache[val]; @@ -240,10 +185,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(Type* val) mTypeMapCache[val] = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(mTypeMapCache[val]); - } - mTypeMapCache[val]->becomeInternalizedOf(val, *this); return mTypeMapCache[val]; @@ -270,10 +211,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(InstanceRef val) mInstanceCache[val] = new PyObjSnapshot(mGraph); - if (mGraph) { - mGraph->registerSnapshot(mInstanceCache[val]); - } - mInstanceCache[val]->becomeInternalizedOf(val, *this); return mInstanceCache[val]; diff --git a/typed_python/PyPyObjGraphSnapshot.cpp b/typed_python/PyPyObjGraphSnapshot.cpp index 2bd26c047..33c3031cb 100644 --- a/typed_python/PyPyObjGraphSnapshot.cpp +++ b/typed_python/PyPyObjGraphSnapshot.cpp @@ -23,6 +23,9 @@ PyDoc_STRVAR(PyPyObjGraphSnapshot_doc, PyMethodDef PyPyObjGraphSnapshot_methods[] = { {"extractTypes", (PyCFunction)PyPyObjGraphSnapshot::extractTypes, METH_VARARGS | METH_KEYWORDS, NULL}, + {"getObjects", (PyCFunction)PyPyObjGraphSnapshot::getObjects, METH_VARARGS | METH_KEYWORDS, NULL}, + {"internalize", (PyCFunction)PyPyObjGraphSnapshot::internalize, METH_VARARGS | METH_KEYWORDS, NULL}, + {"hashToObject", (PyCFunction)PyPyObjGraphSnapshot::hashToObject, METH_VARARGS | METH_KEYWORDS, NULL}, {NULL} /* Sentinel */ }; @@ -45,6 +48,61 @@ PyObject* PyPyObjGraphSnapshot::newPyObjGraphSnapshot(PyObjGraphSnapshot* g, boo return (PyObject*)self; } +/* static */ +PyObject* PyPyObjGraphSnapshot::hashToObject(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {"hash", NULL}; + + PyObject* hashPyobj = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", (char**)kwlist, &hashPyobj)) { + throw PythonExceptionSet(); + } + + if (!PyUnicode_Check(hashPyobj)) { + throw std::runtime_error("Invalid sha hash argument"); + } + + const char* hash = PyUnicode_AsUTF8(hashPyobj); + + ShaHash h = ShaHash::fromHexDigest(hash); + + PyObjSnapshot* s = self->mGraphSnapshot->snapshotForHash(h); + + if (!s) { + return incref(Py_None); + } + + return PyPyObjSnapshot::newPyObjSnapshot(s, graph); + }); +} + +/* static */ +PyObject* PyPyObjGraphSnapshot::internalize(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {"sourceGraph", NULL}; + + PyObject* pySourceGraph = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", (char**)kwlist, &pySourceGraph)) { + throw PythonExceptionSet(); + } + + if (pySourceGraph->ob_type != &PyType_PyObjGraphSnapshot) { + throw std::runtime_error("Expected a PyObjGraphSnapshot for 'sourceGraph'"); + } + + PyPyObjGraphSnapshot* sourceGraph = (PyPyObjGraphSnapshot*)pySourceGraph; + + self->mGraphSnapshot->internalize(*sourceGraph->mGraphSnapshot, false); + return incref(Py_None); + }); +} + /* static */ PyObject* PyPyObjGraphSnapshot::extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs) { PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; @@ -68,6 +126,27 @@ PyObject* PyPyObjGraphSnapshot::extractTypes(PyObject* graph, PyObject *args, Py }); } +/* static */ +PyObject* PyPyObjGraphSnapshot::getObjects(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + throw PythonExceptionSet(); + } + + PyObjectStealer res(PySet_New(NULL)); + + for (auto o: self->mGraphSnapshot->getObjects()) { + PySet_Add(res, PyPyObjSnapshot::newPyObjSnapshot(o, graph)); + } + + return incref(res); + }); +} + /* static */ PyObject* PyPyObjGraphSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kwargs) { @@ -101,8 +180,9 @@ PyObject* PyPyObjGraphSnapshot::tp_repr(PyObject *selfObj) { /* static */ int PyPyObjGraphSnapshot::init(PyPyObjGraphSnapshot *self, PyObject *args, PyObject *kwargs) { - PyErr_Format(PyExc_RuntimeError, "PyObjGraphSnapshot cannot be initialized directly"); - return -1; + self->mGraphSnapshot = new PyObjGraphSnapshot(); + self->mOwnsSnapshot = true; + return 0; } diff --git a/typed_python/PyPyObjGraphSnapshot.hpp b/typed_python/PyPyObjGraphSnapshot.hpp index 5b1621219..73eedbeae 100644 --- a/typed_python/PyPyObjGraphSnapshot.hpp +++ b/typed_python/PyPyObjGraphSnapshot.hpp @@ -30,8 +30,14 @@ class PyPyObjGraphSnapshot { static void dealloc(PyPyObjGraphSnapshot *self); + static PyObject* hashToObject(PyObject* graph, PyObject *args, PyObject *kwargs); + + static PyObject* internalize(PyObject* graph, PyObject *args, PyObject *kwargs); + static PyObject* extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs); + static PyObject* getObjects(PyObject* graph, PyObject *args, PyObject *kwargs); + static PyObject *new_(PyTypeObject *type, PyObject *args, PyObject *kwargs); static PyObject* newPyObjGraphSnapshot(PyObjGraphSnapshot* o, bool ownsIt); diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index 024a4e6dd..77217c358 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -68,6 +68,7 @@ void PyPyObjSnapshot::dealloc(PyPyObjSnapshot *self) decref(self->mKeys); decref(self->mGraph); decref(self->mByKey); + decref(self->mNamedElements); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -79,6 +80,7 @@ PyObject* PyPyObjSnapshot::newPyObjSnapshot(PyObjSnapshot* g, PyObject* pyGraphP self->mElements = nullptr; self->mKeys = nullptr; self->mByKey = nullptr; + self->mNamedElements = nullptr; return (PyObject*)self; } @@ -96,6 +98,7 @@ PyObject* PyPyObjSnapshot::new_(PyTypeObject *type, PyObject *args, PyObject *kw self->mKeys = nullptr; self->mGraph = nullptr; self->mByKey = nullptr; + self->mNamedElements = nullptr; } return (PyObject*)self; @@ -188,13 +191,14 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { if (!pySnap->mKeys) { pySnap->mKeys = PyList_New(0); for (auto p: obj->keys()) { - PyList_Append( - pySnap->mKeys, + PyObjectStealer subPySnap( PyPyObjSnapshot::newPyObjSnapshot( p, p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr ) ); + + PyList_Append(pySnap->mKeys, subPySnap); } } @@ -207,18 +211,44 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { for (long k = 0; k < obj->keys().size(); k++) { PyObjSnapshot* p = obj->elements()[k]; + PyObjectStealer subPySnap( + PyPyObjSnapshot::newPyObjSnapshot( + p, + p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr + ) + ); + PyDict_SetItem( pySnap->mByKey, obj->keys()[k]->getPyObj(), + subPySnap + ); + } + } + + return incref(pySnap->mByKey); + } + + if (attr == "namedElements") { + if (!pySnap->mNamedElements) { + pySnap->mNamedElements = PyDict_New(); + for (auto nameAndSnap: obj->namedElements()) { + PyObjSnapshot* p = nameAndSnap.second; + PyObjectStealer newSnap( PyPyObjSnapshot::newPyObjSnapshot( p, p->getGraph() == obj->getGraph() ? pySnap->mGraph : nullptr ) ); + PyDict_SetItemString( + pySnap->mNamedElements, + nameAndSnap.first.c_str(), + newSnap + ); } } - return incref(pySnap->mByKey); + return incref(pySnap->mNamedElements); } return PyObject_GenericGetAttr(selfObj, attrName); diff --git a/typed_python/PyPyObjSnapshot.hpp b/typed_python/PyPyObjSnapshot.hpp index 1c4a78a74..33515145d 100644 --- a/typed_python/PyPyObjSnapshot.hpp +++ b/typed_python/PyPyObjSnapshot.hpp @@ -31,6 +31,7 @@ class PyPyObjSnapshot { PyObject* mElements; PyObject* mKeys; PyObject* mByKey; + PyObject* mNamedElements; static PyObject* tp_repr(PyObject *selfObj); diff --git a/typed_python/ShaHash.hpp b/typed_python/ShaHash.hpp index 02a2ead61..a74335cb9 100644 --- a/typed_python/ShaHash.hpp +++ b/typed_python/ShaHash.hpp @@ -122,6 +122,29 @@ class ShaHash { return res; } + static ShaHash fromHexDigest(std::string s) { + if (s.size() != sizeof(ShaHash) * 2 + 2) { + throw std::runtime_error("Incorrect size for hash-digest"); + } + + ShaHash res; + uint8_t* srcDataPtr = (uint8_t*)&s[2]; + uint8_t* destDataPtr = (uint8_t*)&res; + + auto fromHex = [&](uint8_t u) { + if (u >= '0' && u <= '9') { return u - '0'; } + if (u >= 'a' && u <= 'f') { return 10 + u - 'a'; } + if (u >= 'A' && u <= 'F') { return 10 + u - 'A'; } + return 0; + }; + + for (int i = 0; i < sizeof(ShaHash); i++) { + destDataPtr[i] = fromHex(srcDataPtr[i * 2]) * 16 + fromHex(srcDataPtr[i * 2 + 1]); + } + + return res; + } + // create a 'poison' sha hash that, like a nan in float-land, // always produces another poison sha hash when added. We can use this // to indicate that a hash is 'bad' in some dimension. diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 17ba7d37d..406f43eb5 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -415,204 +415,243 @@ void Type::attemptToResolve() { return; } - // do the entire type resolution process while holding the GIL - PyEnsureGilAcquired getTheGil; + if (::getenv("TP_NEW_INTERNING") && ::getenv("TP_NEW_INTERNING")[0]) { + // do the entire type resolution process while holding the GIL + PyEnsureGilAcquired getTheGil; + + // compute a snapshot starting with us as the root + PyObjGraphSnapshot graph(this); + + // build a map from our existing types into the graph since the graph is going + // to change + std::map typeToSnap; + for (auto snap: graph.getObjects()) { + if (snap->getType()) { + typeToSnap[snap->getType()] = snap; + } + } + if (typeToSnap.find(this) == typeToSnap.end()) { + throw std::runtime_error("Somehow, we're not in the graph?"); + } - // compute a snapshot starting with us as the root - PyObjGraphSnapshot graph(this); + // take the graph and remove any forwards from it. We will now have a graph where + // no object points directly to a forward anymore. Each forward will point to its + // resolution, and every PyObjSnapshot has a link back to the original object or + // type that it came from. + graph.resolveForwards(); + + // now take every non-forward node and make a version of it in the internal graph, + // if it doesn't already exist. Then rehydrate it. At this point, we have an interned + // copy of every single type and pyobject that was present in this graph. + graph.internalize(); + + // now we need to link the new graph back to our types. + // Every forward Type in our current graph needs to be pointed to the resolved type. + for (auto typeAndSnap: typeToSnap) { + PyObjSnapshot* internalizedSnap = graph.internal().snapshotForHash( + graph.hashFor(typeAndSnap.second) + ); - // take the graph and remove any forwards from it. We will now have a graph where - // no object points directly to a forward anymore. Each forward will point to its - // resolution, and every PyObjSnapshot has a link back to the original object or - // type that it came from. - graph.resolveForwards(); + if (!internalizedSnap) { + throw std::runtime_error("Somehow our Type didn't map to a type in the internalized graph"); + } - // now take every non-forward node and make a version of it in the internal graph, - // if it doesn't already exist. Then rehydrate it. - unresolved object, then we didn't need to update it and it'll be hydrated as normal - graph.internalize(); + Type* target = internalizedSnap->getType(); - // // now type in 'graph' can determine which fully realized type it maps to. + if (typeAndSnap.first->m_forward_resolves_to) { + if (typeAndSnap.first->m_forward_resolves_to != target) { + throw std::runtime_error("This type is already resolved and not to our target!"); + } + } else { + typeAndSnap.first->m_forward_resolves_to = target; + } + } - std::set typesNeedingResolution; - reachableUnresolvedTypes(this, typesNeedingResolution); + if (!m_forward_resolves_to) { + throw std::runtime_error("Somehow, we didn't resolve???"); + } + } else { + std::set typesNeedingResolution; + reachableUnresolvedTypes(this, typesNeedingResolution); - std::set existingReferencedTypes; + std::set existingReferencedTypes; - for (auto t: typesNeedingResolution) { - if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { - throw std::runtime_error( - "Forward defined as " + t->nameWithModule() + " has not been defined." - ); + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget() && !((Forward*)t)->lambdaDefinition()) { + throw std::runtime_error( + "Forward defined as " + t->nameWithModule() + " has not been defined." + ); + } } - } - for (auto t: typesNeedingResolution) { - if (t->isForward() && !((Forward*)t)->getTarget()) { - ((Forward*)t)->define( - ((Forward*)t)->lambdaDefinition() - ); + for (auto t: typesNeedingResolution) { + if (t->isForward() && !((Forward*)t)->getTarget()) { + ((Forward*)t)->define( + ((Forward*)t)->lambdaDefinition() + ); + } } - } - // for each type that we have defined in our graph, what type are we mapping it to? - // for Forwards of primitive types like ListOf or TupleOf that can see themselves, this - // this will be a 'named copy' of the type depending on the Forward name. - std::map resolutionMapping; - // for each type we end up defining, which type did it come from - std::map resolutionSource; + // for each type that we have defined in our graph, what type are we mapping it to? + // for Forwards of primitive types like ListOf or TupleOf that can see themselves, this + // this will be a 'named copy' of the type depending on the Forward name. + std::map resolutionMapping; - // allocate new target type bodies for all regular types in the graph - // that are forward declared. - for (auto t: typesNeedingResolution) { - if (!t->isForward()) { - // we're resolving to this type directly - resolutionMapping[t] = t->cloneForForwardResolution(); - resolutionSource[resolutionMapping[t]] = t; + // for each type we end up defining, which type did it come from + std::map resolutionSource; + + // allocate new target type bodies for all regular types in the graph + // that are forward declared. + for (auto t: typesNeedingResolution) { + if (!t->isForward()) { + // we're resolving to this type directly + resolutionMapping[t] = t->cloneForForwardResolution(); + resolutionSource[resolutionMapping[t]] = t; + } } - } - // now ensure that the target for any forward is the underlying type - // note that we don't need a source for any of these since nobody will end up - // actually having a forward in their graph - for (auto t: typesNeedingResolution) { - if (t->isForward()) { - Forward* f = (Forward*)t; + // now ensure that the target for any forward is the underlying type + // note that we don't need a source for any of these since nobody will end up + // actually having a forward in their graph + for (auto t: typesNeedingResolution) { + if (t->isForward()) { + Forward* f = (Forward*)t; - Type* tgt = f->getTargetTransitive(); + Type* tgt = f->getTargetTransitive(); - if (!tgt) { - throw std::runtime_error("somehow this forward doesn't have a target"); - } + if (!tgt) { + throw std::runtime_error("somehow this forward doesn't have a target"); + } - if (typesNeedingResolution.find(tgt) == typesNeedingResolution.end()) { - resolutionMapping[t] = tgt; - existingReferencedTypes.insert(tgt); - } else { - // resolve this forward to whatever its target resolves to - resolutionMapping[t] = resolutionMapping[tgt]; + if (typesNeedingResolution.find(tgt) == typesNeedingResolution.end()) { + resolutionMapping[t] = tgt; + existingReferencedTypes.insert(tgt); + } else { + // resolve this forward to whatever its target resolves to + resolutionMapping[t] = resolutionMapping[tgt]; + } } } - } - - // copy all the types. At this point, they all know that they are not forwards - // but none of them is ready to be identity-hashed yet. They all will be in a state of - // where m_needs_post_init is true. - for (auto typeAndSource: resolutionSource) { - typeAndSource.first->initializeFrom(typeAndSource.second); - typeAndSource.first->updateInternalTypePointers(resolutionMapping); - } - // allow each Function type to build CompilerVisiblePythonObjects for its globals - // after this, any FunctionGlobals will refer to Constants not cells - for (auto typeAndSource: resolutionSource) { - if (typeAndSource.first->isFunction()) { - ((Function*)typeAndSource.first)->internalizeConstants(resolutionMapping); + // copy all the types. At this point, they all know that they are not forwards + // but none of them is ready to be identity-hashed yet. They all will be in a state of + // where m_needs_post_init is true. + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->initializeFrom(typeAndSource.second); + typeAndSource.first->updateInternalTypePointers(resolutionMapping); } - } - // cause each type to recompute its name. We have to do this in a single pass - // without any types being aware of their final name before we run the post initialize - // step. Otherwise, it's possible for the names to depend on the order of initialization - // and we want them to be stable. - for (auto typeAndSource: resolutionSource) { - typeAndSource.first->recomputeName(); - } + // allow each Function type to build CompilerVisiblePythonObjects for its globals + // after this, any FunctionGlobals will refer to Constants not cells + for (auto typeAndSource: resolutionSource) { + if (typeAndSource.first->isFunction()) { + ((Function*)typeAndSource.first)->internalizeConstants(resolutionMapping); + } + } - // cause each type to post-initialize itself, which lets it update internal bytecounts and - // default initialization flags - bool anyUpdated = true; - size_t passCt = 0; - while (anyUpdated) { - anyUpdated = false; + // cause each type to recompute its name. We have to do this in a single pass + // without any types being aware of their final name before we run the post initialize + // step. Otherwise, it's possible for the names to depend on the order of initialization + // and we want them to be stable. for (auto typeAndSource: resolutionSource) { - if (typeAndSource.first->postInitialize()) { - anyUpdated = true; + typeAndSource.first->recomputeName(); + } + + // cause each type to post-initialize itself, which lets it update internal bytecounts and + // default initialization flags + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto typeAndSource: resolutionSource) { + if (typeAndSource.first->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > resolutionSource.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); } } - passCt += 1; - // we can run this algorithm until all type sizes have stabilized. Conceivably we - // could introduce an error that would cause this to not converge - this should - // detect that. - if (passCt > resolutionSource.size() * 2 + 10) { - throw std::runtime_error("Type size graph is not stabilizing."); + // let each type update any internal caches it might need before it gets instantiated + for (auto typeAndSource: resolutionSource) { + typeAndSource.first->finalizeType(); } - } - // let each type update any internal caches it might need before it gets instantiated - for (auto typeAndSource: resolutionSource) { - typeAndSource.first->finalizeType(); - } + // std::cout << "resolving group:\n"; + // for (auto typeAndSource: resolutionSource) { + // std::cout << " " << TypeOrPyobj(typeAndSource.first).name() << " from " << TypeOrPyobj(typeAndSource.second).name() << "\n"; + // } - // std::cout << "resolving group:\n"; - // for (auto typeAndSource: resolutionSource) { - // std::cout << " " << TypeOrPyobj(typeAndSource.first).name() << " from " << TypeOrPyobj(typeAndSource.second).name() << "\n"; - // } + // now internalize the types by their hash. For each type, we compute a hash + // and then look to see if we've seen it before. We build a lookup table from + // each existing type to the internalized type, and then do the same process we did + // above to map any roots across. - // now internalize the types by their hash. For each type, we compute a hash - // and then look to see if we've seen it before. We build a lookup table from - // each existing type to the internalized type, and then do the same process we did - // above to map any roots across. + std::map resolvedToInternal; // each 'resolved' type what should it be replaced with - std::map resolvedToInternal; // each 'resolved' type what should it be replaced with + std::set newTypes; + std::set redundantTypes; - std::set newTypes; - std::set redundantTypes; + for (auto typeAndSource: resolutionSource) { + ShaHash h = typeAndSource.first->identityHash(); - for (auto typeAndSource: resolutionSource) { - ShaHash h = typeAndSource.first->identityHash(); + auto it = mInternalizedIdentityHashToType.find(h); + if (it == mInternalizedIdentityHashToType.end()) { + // this is a new type. We're going to put it in the memo and + // update it so that it ponts + newTypes.insert(typeAndSource.first); + } else { + resolvedToInternal[typeAndSource.first] = it->second; + redundantTypes.insert(typeAndSource.first); + } + } - auto it = mInternalizedIdentityHashToType.find(h); - if (it == mInternalizedIdentityHashToType.end()) { - // this is a new type. We're going to put it in the memo and - // update it so that it ponts - newTypes.insert(typeAndSource.first); - } else { - resolvedToInternal[typeAndSource.first] = it->second; - redundantTypes.insert(typeAndSource.first); + // update all the new types to no longer look at the redundant types + for (auto t: newTypes) { + t->updateInternalTypePointers(resolvedToInternal); + mInternalizedIdentityHashToType[t->identityHash()] = t; + resolvedToInternal[t] = t; } - } - // update all the new types to no longer look at the redundant types - for (auto t: newTypes) { - t->updateInternalTypePointers(resolvedToInternal); - mInternalizedIdentityHashToType[t->identityHash()] = t; - resolvedToInternal[t] = t; - } + for (auto t: redundantTypes) { + t->markRedundant(); + } - for (auto t: redundantTypes) { - t->markRedundant(); - } + // tell each source type which type it actually resolves to. We're holding the GIL so + // nobody should see anything until we finish this process. + for (auto typeAndTarget: resolutionMapping) { + auto it = resolvedToInternal.find(typeAndTarget.second); + if (it == resolvedToInternal.end()) { + // this must be an existing type + if (existingReferencedTypes.find(typeAndTarget.second) == existingReferencedTypes.end()) { + throw std::runtime_error( + "Failed to get an internalization of Type " + + typeAndTarget.second->name() + " and it wasn't a pre-existing type either." + ); + } - // tell each source type which type it actually resolves to. We're holding the GIL so - // nobody should see anything until we finish this process. - for (auto typeAndTarget: resolutionMapping) { - auto it = resolvedToInternal.find(typeAndTarget.second); - if (it == resolvedToInternal.end()) { - // this must be an existing type - if (existingReferencedTypes.find(typeAndTarget.second) == existingReferencedTypes.end()) { - throw std::runtime_error( - "Failed to get an internalization of Type " - + typeAndTarget.second->name() + " and it wasn't a pre-existing type either." - ); + // we resolve to it directly + typeAndTarget.first->m_forward_resolves_to = typeAndTarget.second; + } else { + typeAndTarget.first->m_forward_resolves_to = it->second; } - - // we resolve to it directly - typeAndTarget.first->m_forward_resolves_to = typeAndTarget.second; - } else { - typeAndTarget.first->m_forward_resolves_to = it->second; } - } - if (!m_forward_resolves_to) { - throw std::runtime_error("Somehow, we didn't resolve???"); - } + if (!m_forward_resolves_to) { + throw std::runtime_error("Somehow, we didn't resolve???"); + } - for (auto typeAndTarget: resolutionMapping) { - if (typeAndTarget.first->isForwardDefined()) { - typeAndTarget.second->addForwardDefinition(typeAndTarget.first); + for (auto typeAndTarget: resolutionMapping) { + if (typeAndTarget.first->isForwardDefined()) { + typeAndTarget.second->addForwardDefinition(typeAndTarget.first); + } } } } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index 3d357c532..d31de9745 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -637,6 +637,14 @@ class Type { return m_base; } + size_t typeLevel() const { + if (!m_base) { + return 0; + } + + return m_base->typeLevel() + 1; + } + // are these types equivalent up to identity? This should be // preferred over t1 == t2 for comparing types in cases where // we don't need _exact_ identity equiality, because there are @@ -1058,13 +1066,15 @@ class Type { static std::map mInternalizedIdentityHashToType; - void setSnapshot(PyObjSnapshot* snapshot); +public: + void setSnapshot(PyObjSnapshot* snapshot) { + mSnapshot = snapshot; + } PyObjSnapshot* getSnapshot() const { return mSnapshot; } -public: void addForwardDefinition(Type* t) { if (!t->isForwardDefined()) { throw std::runtime_error("Can't add a non-forward defined type as a reverse Forward link"); diff --git a/typed_python/_types.cpp b/typed_python/_types.cpp index 5c37d789a..545ab6b06 100644 --- a/typed_python/_types.cpp +++ b/typed_python/_types.cpp @@ -3840,9 +3840,14 @@ PyInit__types(void) if (module == NULL) return NULL; - // initialize a couple of global references to things in typed_python.internals - PythonObjectOfType::AnyPyObject(); - PythonObjectOfType::AnyPyType(); + try { + // initialize a couple of global references to things in typed_python.internals + PythonObjectOfType::AnyPyObject(); + PythonObjectOfType::AnyPyType(); + } catch (std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + return NULL; + } if (PyType_Ready(&PyType_Slab) < 0) { return NULL; diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index 17d1120b5..2c645dbc8 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -1,7 +1,7 @@ import numpy from typed_python import ListOf -from typed_python._types import PyObjSnapshot, _enableTypeAutoresolution +from typed_python._types import PyObjSnapshot, PyObjGraphSnapshot, _enableTypeAutoresolution class DisableForwardAutoresolve: @@ -251,3 +251,32 @@ def test_rehydration_preserves_sha_hashing(): with DisableForwardAutoresolve(): graphIdempotenceCheck(int) graphIdempotenceCheck(ListOf(int)) + + +def test_snapshot_of_object(): + assert PyObjSnapshot.create(object).kind == 'NamedPyObject' + assert PyObjSnapshot.create(type).kind == 'NamedPyObject' + + +def test_snapshot_of_types_basic(): + snap = PyObjSnapshot.create(ListOf(float), False) + assert snap.kind == 'ListOfType' + assert snap.element_type.kind == 'PrimitiveType' + + newGraph = PyObjGraphSnapshot() + newGraph.internalize(snap.graph) + newsnap = newGraph.hashToObject(snap.shaHash) + assert newsnap.kind == 'ListOfType' + assert newsnap.type.__typed_python_category__ == 'ListOf' + + +def test_snapshot_of_types_with_object(): + snap = PyObjSnapshot.create(ListOf(object), False) + assert snap.kind == 'ListOfType' + assert snap.element_type.kind == 'PythonObjectOfTypeType' + + newGraph = PyObjGraphSnapshot() + newGraph.internalize(snap.graph) + newsnap = newGraph.hashToObject(snap.shaHash) + assert newsnap.kind == 'ListOfType' + assert newsnap.type.__typed_python_category__ == 'ListOf' From 96553f4d146ce6066d0e6341292fc958015014e1 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 4 Aug 2023 03:46:02 +0000 Subject: [PATCH 80/83] more progress --- typed_python/CompositeType.hpp | 4 ++ typed_python/HeldClassType.hpp | 61 ++++++++++++++++++- typed_python/Instance.cpp | 2 + typed_python/PyClassInstance.cpp | 18 +++++- typed_python/PyInstance.cpp | 1 + typed_python/PyObjGraphSnapshot.cpp | 30 +++++++-- typed_python/PyObjGraphSnapshot.hpp | 7 +++ typed_python/PyObjRehydrator.cpp | 46 ++++++++++---- typed_python/PyObjSnapshot.cpp | 21 +++++-- typed_python/PyObjSnapshot.hpp | 30 +++++++-- typed_python/PyObjSnapshotGroupSearch.hpp | 15 ++++- typed_python/PyPyObjGraphSnapshot.cpp | 26 ++++++-- typed_python/PyPyObjGraphSnapshot.hpp | 2 + typed_python/Type.cpp | 16 +++-- typed_python/Type.hpp | 17 +++++- typed_python/__init__.py | 2 +- .../compiler/native_compiler/native_ast.py | 5 +- typed_python/internals.py | 5 +- typed_python/py_obj_snapshot_test.py | 14 ++++- 19 files changed, 275 insertions(+), 47 deletions(-) diff --git a/typed_python/CompositeType.hpp b/typed_python/CompositeType.hpp index e26acec3a..1733baabb 100644 --- a/typed_python/CompositeType.hpp +++ b/typed_python/CompositeType.hpp @@ -254,6 +254,10 @@ class CompositeType : public Type { m_byte_offsets.clear(); + for (long k = 0; k < m_names.size(); k++) { + m_nameToIndex[m_names[k]] = k; + } + for (auto t: m_types) { m_byte_offsets.push_back(size); size += t->bytecount(); diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 43e9a7570..95c53e467 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -447,6 +447,46 @@ class HeldClass : public Type { m_classType = clsType; } + void initializeDuringDeserialization( + std::string inName, + const std::vector& bases, + bool isFinal, + const std::vector& members, + const std::map& memberFunctions, + const std::map& staticFunctions, + const std::map& propertyFunctions, + const std::map& classMembers, + const std::map& classMethods, + + const std::vector& own_members, + const std::map& own_memberFunctions, + const std::map& own_staticFunctions, + const std::map& own_propertyFunctions, + const std::map& own_classMembers, + const std::map& own_classMethods, + + Class* clsType + ) { + m_name = inName; + m_bases = bases; + m_is_final = isFinal; + m_own_members = own_members; + m_own_memberFunctions = own_memberFunctions; + m_own_staticFunctions = own_staticFunctions; + m_own_propertyFunctions = own_propertyFunctions; + m_own_classMembers = own_classMembers; + m_own_classMethods = own_classMethods; + + m_members = members; + m_memberFunctions = memberFunctions; + m_staticFunctions = staticFunctions; + m_propertyFunctions = propertyFunctions; + m_classMembers = classMembers; + m_classMethods = classMethods; + + m_classType = clsType; + } + const char* docConcrete() { return HeldClass_doc; } @@ -951,10 +991,20 @@ class HeldClass : public Type { // this function can be called multiple times, which happens after the class is // deserialized, since we may be rebuilding the function globals. void mergeOwnFunctionsIntoInheritanceTree() { + if (m_classMembers.size() + || m_memberFunctions.size() + || m_classMethods.size() + || m_propertyFunctions.size() + || m_staticFunctions.size() + ) { + return; + } + m_classMembers.clear(); m_memberFunctions.clear(); m_classMethods.clear(); m_propertyFunctions.clear(); + m_staticFunctions.clear(); for (HeldClass* base: m_mro) { for (auto nameAndObj: base->m_own_classMembers) { @@ -987,13 +1037,18 @@ class HeldClass : public Type { throw std::runtime_error("Somehow " + m_name + " doesn't have itself as MRO 0"); } + // build our own method resolution table directly from our parents. + // we only have to do this if we're forward defined since the type + // resolution process will just copy the functions over when it resolves + // them + if (!m_is_forward_defined) { + mergeOwnFunctionsIntoInheritanceTree(); + } + if (m_is_forward_defined) { return; } - // build our own method resolution table directly from our parents. - mergeOwnFunctionsIntoInheritanceTree(); - std::set membersSoFar; //only one base class can have members diff --git a/typed_python/Instance.cpp b/typed_python/Instance.cpp index a418c5429..c9e9e4694 100644 --- a/typed_python/Instance.cpp +++ b/typed_python/Instance.cpp @@ -79,6 +79,8 @@ Instance::Instance(const Instance& other) : mLayout(other.mLayout) { } Instance::Instance(const InstanceRef& other) { + mLayout = nullptr; + Type* t = other.type(); instance_ptr p = other.data(); diff --git a/typed_python/PyClassInstance.cpp b/typed_python/PyClassInstance.cpp index e8744d8d2..8f1b914cb 100644 --- a/typed_python/PyClassInstance.cpp +++ b/typed_python/PyClassInstance.cpp @@ -738,7 +738,16 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT PyObjectStealer memberFunctions(PyDict_New()); for (auto p: classT->isForwardDefined() ? classT->getOwnMemberFunctions() : classT->getMemberFunctions()) { - PyDict_SetItemString(memberFunctions, p.first.c_str(), typePtrToPyTypeRepresentation(p.second)); + typePtrToPyTypeRepresentation(p.second); + if (!p.second->isTypeObjReady()) { + throw std::runtime_error("TypeObj for " + p.second->name() + " is not ready"); + } + + PyDict_SetItemString( + memberFunctions, + p.first.c_str(), + typePtrToPyTypeRepresentation(p.second) + ); if (!classT->isForwardDefined()) { // TODO: clean this up. I'm not sure what we are guarding against - is it @@ -809,6 +818,13 @@ void PyClassInstance::mirrorTypeInformationIntoPyTypeConcrete(Class* classT, PyT PyDict_SetItemString(pyType->tp_dict, "StaticMemberFunctions", staticMemberFunctions); for (auto nameAndObj: classT->isForwardDefined() ? classT->getOwnStaticFunctions() : classT->getStaticFunctions()) { + + typePtrToPyTypeRepresentation(nameAndObj.second); + if (!nameAndObj.second->isTypeObjReady()) { + asm("int3"); + throw std::runtime_error("TypeObj for " + nameAndObj.second->name() + " is not ready"); + } + PyDict_SetItemString( staticMemberFunctions, nameAndObj.first.c_str(), typePtrToPyTypeRepresentation(nameAndObj.second) diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 3a13ddd3b..8b698f82b 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1543,6 +1543,7 @@ void PyInstance::finalizePyTypeObjectPhase1(Type* inType, PyTypeObject* pyType) pyType->tp_methods = typeMethods(inType); PyType_Ready(pyType); + inType->markTypeObjReady(); } void PyInstance::finalizePyTypeObjectPhase2(Type* inType, PyTypeObject* pyType) { diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index 6fed9f10f..33aa4443e 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -58,9 +58,7 @@ void PyObjGraphSnapshot::resolveForwards() { if (o->getKind() == PyObjSnapshot::Kind::ForwardType) { o->pointForwardToFinalType(); modified.insert(o); - } - - if (o->willBeATpType()) { + } else if (o->willBeATpType()) { if (o->markTypeNotFwdDefined()) { modified.insert(o); } @@ -137,6 +135,7 @@ void PyObjGraphSnapshot::internalize(PyObjGraphSnapshot& otherGraph, bool markIn } catch(...) { // undo this since the state is somehow corrupt for (auto hashAndSkeleton: skeletons) { + mObjectsByIndex.pop_back(); mObjects.erase(hashAndSkeleton.second.second); mHashToSnap.erase(hashAndSkeleton.first); } @@ -154,6 +153,7 @@ PyObjSnapshot* PyObjGraphSnapshot::createSkeleton(ShaHash h) { mHashToSnap[h] = snap; mObjects.insert(snap); + mObjectsByIndex.push_back(snap); return snap; } @@ -174,6 +174,17 @@ ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { b->count(snap->getInstance().data()) ); } + if (snap->getInstance().type()->isString()) { + static StringType* b = StringType::Make(); + + std::string s = b->toUtf8String(snap->getInstance().data()); + + return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + + ShaHash::SHA1( + &s[0], + s.size() + ); + } throw std::runtime_error( "Can't hash a PyObjSnapshot Instance of type " + snap->getInstance().type()->name() ); @@ -280,7 +291,8 @@ ShaHash PyObjGraphSnapshot::hashFor(PyObjSnapshot* snap) { void PyObjGraphSnapshot::computeHashesFor(const std::unordered_set& group) { PyObjSnapshot* root = pickRootFor(group); - // order our snapshots lexically + // order our snapshots by the order in which we would + // visit them std::unordered_map snapsSeen; std::vector snapsInOrder; @@ -351,6 +363,14 @@ void PyObjGraphSnapshot::installSnapHash(PyObjSnapshot* snap, ShaHash h) { } PyObjSnapshot* PyObjGraphSnapshot::pickRootFor(const std::unordered_set& group) { + if (!group.size()) { + throw std::runtime_error("Can't pick a root for an empty group"); + } + + if (group.size() == 1) { + return *group.begin(); + } + // pick a 'root' node by computing a shallow hash and taking the first value for which // there is only one hash std::unordered_map curHashes; @@ -358,6 +378,8 @@ PyObjSnapshot* PyObjGraphSnapshot::pickRootFor(const std::unordered_set& getObjectsByIndex() const { + return mObjectsByIndex; + } + ShaHash hashFor(PyObjSnapshot* snap); PyObjSnapshot* snapshotForHash(ShaHash h); @@ -106,6 +111,8 @@ class PyObjGraphSnapshot { // all of our objects std::unordered_set mObjects; + std::vector mObjectsByIndex; + // for each snapshot, a hash std::unordered_map mSnapToHash; diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index 85667ebd0..d3b1a166f 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -369,7 +369,9 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { throw std::runtime_error("Corrupt PyObjSnapshot::Value"); } - rehydrate(snap->mNamedElements["value_instance"]); + // TODO: this needs to be a proper rehydration when we are + // able to fully model 'instance' objects. + // rehydrate(snap->mNamedElements["value_instance"]); ((Value*)snap->mType)->initializeDuringDeserialization( snap->mNamedElements["value_instance"]->mInstance @@ -581,6 +583,20 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { getNamedBundle(snap, "cls_classmembers", classMembers); getNamedBundle(snap, "cls_classmethods", classMethods); + std::vector own_members; + std::map own_memberFunctions; + std::map own_staticFunctions; + std::map own_propertyFunctions; + std::map own_classMembers; + std::map own_classMethods; + + getNamedBundle(snap, "cls_own_members", own_members); + getNamedBundle(snap, "cls_own_methods", own_memberFunctions); + getNamedBundle(snap, "cls_own_staticmethods", own_staticFunctions); + getNamedBundle(snap, "cls_own_properties", own_propertyFunctions); + getNamedBundle(snap, "cls_own_classmembers", own_classMembers); + getNamedBundle(snap, "cls_own_classmethods", own_classMethods); + Type* clsType = getNamedElementType(snap, "cls_type"); if (!clsType->isClass()) { throw std::runtime_error("Corrupt PyObjSnapshot.HeldClass"); @@ -596,6 +612,14 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { propertyFunctions, classMembers, classMethods, + + own_members, + own_memberFunctions, + own_staticFunctions, + own_propertyFunctions, + own_classMembers, + own_classMethods, + (Class*)clsType ); return; @@ -948,16 +972,16 @@ void PyObjRehydrator::finalize() { t->typeFinishedBeingDeserializedPhase1(); } - // now that we've set function globals, we need to go over all the classes and - // held classes and make sure the versions of the function types they're actually - // using have the new definitions. Unfortunately, Overload objects are not pointers - // (maybe they should be?) and so when we merge Overloads from base/child classes - // they don't get their globals replaced with the appropriate values - for (auto& t: types) { - if (t && t->isHeldClass()) { - ((HeldClass*)t)->mergeOwnFunctionsIntoInheritanceTree(); - } - } + // // now that we've set function globals, we need to go over all the classes and + // // held classes and make sure the versions of the function types they're actually + // // using have the new definitions. Unfortunately, Overload objects are not pointers + // // (maybe they should be?) and so when we merge Overloads from base/child classes + // // they don't get their globals replaced with the appropriate values + // for (auto& t: types) { + // if (t && t->isHeldClass()) { + // ((HeldClass*)t)->mergeOwnFunctionsIntoInheritanceTree(); + // } + // } for (auto t: types) { t->typeFinishedBeingDeserializedPhase2(); diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index e4eb26c62..0e0905bbb 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -412,12 +412,21 @@ void PyObjSnapshot::becomeInternalizedOf( mNamedElements["cls_type"] = maker.internalize(cls->getClassType()); mNamedElements["cls_bases"] = maker.internalize(cls->getBases()); - mNamedElements["cls_methods"] = maker.internalize(cls->getOwnMemberFunctions()); - mNamedElements["cls_staticmethods"] = maker.internalize(cls->getOwnStaticFunctions()); - mNamedElements["cls_classmethods"] = maker.internalize(cls->getOwnClassMethods()); - mNamedElements["cls_properties"] = maker.internalize(cls->getOwnPropertyFunctions()); - mNamedElements["cls_classmembers"] = maker.internalize(cls->getOwnClassMembers()); - mNamedElements["cls_members"] = maker.internalize(cls->getOwnMembers()); + + mNamedElements["cls_own_methods"] = maker.internalize(cls->getOwnMemberFunctions()); + mNamedElements["cls_own_staticmethods"] = maker.internalize(cls->getOwnStaticFunctions()); + mNamedElements["cls_own_classmethods"] = maker.internalize(cls->getOwnClassMethods()); + mNamedElements["cls_own_properties"] = maker.internalize(cls->getOwnPropertyFunctions()); + mNamedElements["cls_own_classmembers"] = maker.internalize(cls->getOwnClassMembers()); + mNamedElements["cls_own_members"] = maker.internalize(cls->getOwnMembers()); + + mNamedElements["cls_methods"] = maker.internalize(cls->getMemberFunctions()); + mNamedElements["cls_staticmethods"] = maker.internalize(cls->getStaticFunctions()); + mNamedElements["cls_classmethods"] = maker.internalize(cls->getClassMethods()); + mNamedElements["cls_properties"] = maker.internalize(cls->getPropertyFunctions()); + mNamedElements["cls_classmembers"] = maker.internalize(cls->getClassMembers()); + mNamedElements["cls_members"] = maker.internalize(cls->getMembers()); + mNamedInts["cls_is_final"] = cls->isFinal(); return; } diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 91e187588..8df306961 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -502,7 +502,28 @@ class PyObjSnapshot { void rehydrate(); std::string toString() const { - return "PyObjSnapshot." + kindAsString() + "(ix=" + format(mIndexInGraph) + ")"; + std::string inner; + if (mType) { + inner = mType->name(); + } else + if (mName.size()) { + inner = mName; + if (mModuleName.size()) { + inner = mModuleName + "." + mName; + } + } else + if (mPyObject) { + inner = std::string("of type ") + mPyObject->ob_type->tp_name; + } + if (inner.size()) { + inner = ", " + inner; + } + + return "PyObjSnapshot." + kindAsString() + "(ix=" + format(mIndexInGraph) + inner + ")"; + } + + size_t getIndexInGraph() const { + return mIndexInGraph; } template @@ -510,8 +531,7 @@ class PyObjSnapshot { mKind = Kind::InternalBundle; for (auto& nameAndElt: namedElements) { - mNames.push_back(nameAndElt.first); - mElements.push_back(maker.internalize(nameAndElt.second)); + mNamedElements[nameAndElt.first] = maker.internalize(nameAndElt.second); } } @@ -646,11 +666,11 @@ class PyObjSnapshot { template void visitOutbound(const visitor_type& v) { - for (auto e: mElements) { + for (auto& e: mElements) { v(e); } - for (auto e: mKeys) { + for (auto& e: mKeys) { v(e); } diff --git a/typed_python/PyObjSnapshotGroupSearch.hpp b/typed_python/PyObjSnapshotGroupSearch.hpp index 2157c8300..965857b27 100644 --- a/typed_python/PyObjSnapshotGroupSearch.hpp +++ b/typed_python/PyObjSnapshotGroupSearch.hpp @@ -27,6 +27,10 @@ class PyObjSnapshotGroupSearch { } void add(PyObjSnapshot* o) { + if (mSnapToOutGroupIx.find(o) != mSnapToOutGroupIx.end()) { + throw std::runtime_error("We've already seen this element"); + } + pushGroup(o); findAllGroups(); } @@ -41,6 +45,10 @@ class PyObjSnapshotGroupSearch { return; } + if (mSnapToOutGroupIx.find(o) != mSnapToOutGroupIx.end()) { + return; + } + mInAGroup.insert(o); mGroups.push_back( @@ -58,7 +66,9 @@ class PyObjSnapshotGroupSearch { mGroups.back()->insert(o); o->visitOutbound([&](PyObjSnapshot* snap) { - mGroupOutboundEdges.back()->push_back(snap); + if (snap->getGraph() == mGraph) { + mGroupOutboundEdges.back()->push_back(snap); + } }); } @@ -74,6 +84,7 @@ class PyObjSnapshotGroupSearch { mOutGroups.push_back(mGroups.back()); for (auto gElt: *mGroups.back()) { + mSnapToOutGroupIx[gElt] = mOutGroups.size() - 1; mInAGroup.erase(gElt); } @@ -127,4 +138,6 @@ class PyObjSnapshotGroupSearch { // the final groups we ended up with std::vector > > mOutGroups; + + std::unordered_map mSnapToOutGroupIx; }; diff --git a/typed_python/PyPyObjGraphSnapshot.cpp b/typed_python/PyPyObjGraphSnapshot.cpp index 33c3031cb..4d56f318c 100644 --- a/typed_python/PyPyObjGraphSnapshot.cpp +++ b/typed_python/PyPyObjGraphSnapshot.cpp @@ -25,6 +25,7 @@ PyMethodDef PyPyObjGraphSnapshot_methods[] = { {"extractTypes", (PyCFunction)PyPyObjGraphSnapshot::extractTypes, METH_VARARGS | METH_KEYWORDS, NULL}, {"getObjects", (PyCFunction)PyPyObjGraphSnapshot::getObjects, METH_VARARGS | METH_KEYWORDS, NULL}, {"internalize", (PyCFunction)PyPyObjGraphSnapshot::internalize, METH_VARARGS | METH_KEYWORDS, NULL}, + {"resolveForwards", (PyCFunction)PyPyObjGraphSnapshot::resolveForwards, METH_VARARGS | METH_KEYWORDS, NULL}, {"hashToObject", (PyCFunction)PyPyObjGraphSnapshot::hashToObject, METH_VARARGS | METH_KEYWORDS, NULL}, {NULL} /* Sentinel */ }; @@ -103,6 +104,23 @@ PyObject* PyPyObjGraphSnapshot::internalize(PyObject* graph, PyObject *args, PyO }); } +/* static */ +PyObject* PyPyObjGraphSnapshot::resolveForwards(PyObject* graph, PyObject *args, PyObject *kwargs) { + PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; + + return translateExceptionToPyObject([&]() { + static const char *kwlist[] = {NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "", (char**)kwlist)) { + throw PythonExceptionSet(); + } + + self->mGraphSnapshot->resolveForwards(); + + return incref(Py_None); + }); +} + /* static */ PyObject* PyPyObjGraphSnapshot::extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs) { PyPyObjGraphSnapshot* self = (PyPyObjGraphSnapshot*)graph; @@ -116,7 +134,7 @@ PyObject* PyPyObjGraphSnapshot::extractTypes(PyObject* graph, PyObject *args, Py PyObjectStealer res(PySet_New(NULL)); - for (auto o: self->mGraphSnapshot->getObjects()) { + for (auto o: self->mGraphSnapshot->getObjectsByIndex()) { if (o->getType()) { PySet_Add(res, (PyObject*)PyInstance::typeObj(o->getType())); } @@ -137,10 +155,10 @@ PyObject* PyPyObjGraphSnapshot::getObjects(PyObject* graph, PyObject *args, PyOb throw PythonExceptionSet(); } - PyObjectStealer res(PySet_New(NULL)); + PyObjectStealer res(PyList_New(0)); - for (auto o: self->mGraphSnapshot->getObjects()) { - PySet_Add(res, PyPyObjSnapshot::newPyObjSnapshot(o, graph)); + for (auto o: self->mGraphSnapshot->getObjectsByIndex()) { + PyList_Append(res, PyPyObjSnapshot::newPyObjSnapshot(o, graph)); } return incref(res); diff --git a/typed_python/PyPyObjGraphSnapshot.hpp b/typed_python/PyPyObjGraphSnapshot.hpp index 73eedbeae..1fad7c9c1 100644 --- a/typed_python/PyPyObjGraphSnapshot.hpp +++ b/typed_python/PyPyObjGraphSnapshot.hpp @@ -34,6 +34,8 @@ class PyPyObjGraphSnapshot { static PyObject* internalize(PyObject* graph, PyObject *args, PyObject *kwargs); + static PyObject* resolveForwards(PyObject* graph, PyObject *args, PyObject *kwargs); + static PyObject* extractTypes(PyObject* graph, PyObject *args, PyObject *kwargs); static PyObject* getObjects(PyObject* graph, PyObject *args, PyObject *kwargs); diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 406f43eb5..9e38d9d70 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -458,12 +458,18 @@ void Type::attemptToResolve() { Type* target = internalizedSnap->getType(); - if (typeAndSnap.first->m_forward_resolves_to) { - if (typeAndSnap.first->m_forward_resolves_to != target) { - throw std::runtime_error("This type is already resolved and not to our target!"); + if (target && target->isForward()) { + target = ((Forward*)target)->getTargetTransitive(); + } + + if (target) { + if (typeAndSnap.first->m_forward_resolves_to) { + if (typeAndSnap.first->m_forward_resolves_to != target) { + throw std::runtime_error("This type is already resolved and not to our target!"); + } + } else { + typeAndSnap.first->m_forward_resolves_to = target; } - } else { - typeAndSnap.first->m_forward_resolves_to = target; } } diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index d31de9745..b3ecfd1b1 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -604,7 +604,8 @@ class Type { case catSubclassOf: return f(*(SubclassOfType*)this); default: - throw std::runtime_error("Invalid type found"); + asm("int3"); + throw std::runtime_error("Invalid type found: " + format((int)m_typeCategory)); } } @@ -942,6 +943,15 @@ class Type { return m_is_being_deserialized; } + bool isTypeObjReady() { + return m_type_obj_ready; + } + + void markTypeObjReady() { + m_type_obj_ready = true; + } + + void typeFinishedBeingDeserializedPhase1(); void typeFinishedBeingDeserializedPhase2(); @@ -963,9 +973,12 @@ class Type { m_forward_resolves_to(nullptr), m_is_redundant(false), m_is_being_deserialized(false), - mSnapshot(nullptr) + mSnapshot(nullptr), + m_type_obj_ready(false) {} + bool m_type_obj_ready; + PyObjSnapshot* mSnapshot; TypeCategory m_typeCategory; diff --git a/typed_python/__init__.py b/typed_python/__init__.py index 62fb6bb09..4ff4edc2c 100644 --- a/typed_python/__init__.py +++ b/typed_python/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.2.6" +__version__ = "0.2.7" # incremented every time the way we serialize a fully-typed # TP object changes. Consumers who only serialize fully-typed diff --git a/typed_python/compiler/native_compiler/native_ast.py b/typed_python/compiler/native_compiler/native_ast.py index 5dcdf8d28..4df2940f3 100644 --- a/typed_python/compiler/native_compiler/native_ast.py +++ b/typed_python/compiler/native_compiler/native_ast.py @@ -661,7 +661,6 @@ def expr_could_throw(self): couldThrow=expr_could_throw )) - Expression = resolveForwardDefinedType(Expression) NamedCallTarget = resolveForwardDefinedType(NamedCallTarget) Type = resolveForwardDefinedType(Type) @@ -670,6 +669,10 @@ def expr_could_throw(self): Teardown = resolveForwardDefinedType(Teardown) CallTarget = resolveForwardDefinedType(CallTarget) +from typed_python._types import isForwardDefined +print(Expression.__typed_python_category__) +assert not isForwardDefined(Expression) + def ensureExpr(x): if isinstance(x, int): diff --git a/typed_python/internals.py b/typed_python/internals.py index f4694bf52..bd7185f1f 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -544,7 +544,8 @@ def extractCodeObjectNewStatementLineNumbers(codeObject): return res except Exception: # nasty to swallow the exception like this. At least we report it... - import traceback - traceback.print_exc() + print("FAILED in extractCodeObjectNewStatementLineNumbers") + # import traceback + # traceback.print_exc() return [] diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index 2c645dbc8..6ca0a6ebb 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -1,6 +1,6 @@ import numpy -from typed_python import ListOf +from typed_python import ListOf, Alternative, Forward from typed_python._types import PyObjSnapshot, PyObjGraphSnapshot, _enableTypeAutoresolution @@ -280,3 +280,15 @@ def test_snapshot_of_types_with_object(): newsnap = newGraph.hashToObject(snap.shaHash) assert newsnap.kind == 'ListOfType' assert newsnap.type.__typed_python_category__ == 'ListOf' + + +def test_alternative_methods(): + C = Forward(lambda: C) + + X = Alternative("X", x={'c': C}, f=lambda self: 1) + + snap = PyObjSnapshot.create(X, False) + snap2 = PyObjSnapshot.create(snap.pyobj, False) + + assert snap.shaHash == snap2.shaHash + assert snap.pyobj.f.__name__ == 'f' From 89ab2b5e5e7b3f47291ac042d2ba6b2bdf85fb17 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 4 Aug 2023 11:03:39 +0000 Subject: [PATCH 81/83] Fix a few more bugs. --- typed_python/HeldClassType.hpp | 2 +- typed_python/PyObjGraphSnapshot.cpp | 7 +++++ typed_python/PyObjRehydrator.cpp | 27 +++++++++---------- typed_python/PyObjSnapshot.cpp | 9 +++++++ .../compiler/native_compiler/native_ast.py | 4 --- typed_python/internals.py | 1 + 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/typed_python/HeldClassType.hpp b/typed_python/HeldClassType.hpp index 95c53e467..ec6687ad1 100644 --- a/typed_python/HeldClassType.hpp +++ b/typed_python/HeldClassType.hpp @@ -1041,7 +1041,7 @@ class HeldClass : public Type { // we only have to do this if we're forward defined since the type // resolution process will just copy the functions over when it resolves // them - if (!m_is_forward_defined) { + if (m_is_forward_defined) { mergeOwnFunctionsIntoInheritanceTree(); } diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index 33aa4443e..8794553b0 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -194,6 +194,13 @@ ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { } if (snap->getKind() == PyObjSnapshot::Kind::ArbitraryPyObject) { // probably we should try to serialize it? + if (PyType_Check(snap->getPyObj())) { + throw std::runtime_error( + "Can't hash a PyObjSnapshot.ArbitraryPyObject for type " + + std::string(((PyTypeObject*)snap->getPyObj())->tp_name) + ); + } + throw std::runtime_error( "Can't hash a PyObjSnapshot.ArbitraryPyObject of type " + std::string(snap->getPyObj()->ob_type->tp_name) diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index d3b1a166f..dc1d4d649 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -626,6 +626,9 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::FunctionType) { + snap->mType = new Function(); + snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + std::string name = snap->mName; std::string qualname = snap->mQualname; std::string moduleName = snap->mModuleName; @@ -636,9 +639,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { getNamedBundle(snap, "func_overloads", overloads); - snap->mType = new Function(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - ((Function*)snap->mType)->initializeDuringDeserialization( name, qualname, @@ -923,6 +923,16 @@ void PyObjRehydrator::finalize() { finalizeRehydration(n); } + for (auto n: mSnapshots) { + if (n->willBeATpType()) { + if (!n->mType) { + throw std::runtime_error( + "Snapshot " + n->toString() + " doesn't have a Type" + ); + } + } + } + std::vector types; for (auto o: mSnapshots) { if (o->mType) { @@ -972,17 +982,6 @@ void PyObjRehydrator::finalize() { t->typeFinishedBeingDeserializedPhase1(); } - // // now that we've set function globals, we need to go over all the classes and - // // held classes and make sure the versions of the function types they're actually - // // using have the new definitions. Unfortunately, Overload objects are not pointers - // // (maybe they should be?) and so when we merge Overloads from base/child classes - // // they don't get their globals replaced with the appropriate values - // for (auto& t: types) { - // if (t && t->isHeldClass()) { - // ((HeldClass*)t)->mergeOwnFunctionsIntoInheritanceTree(); - // } - // } - for (auto t: types) { t->typeFinishedBeingDeserializedPhase2(); } diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 0e0905bbb..49debba44 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -505,6 +505,15 @@ void PyObjSnapshot::becomeInternalizedOf( mPyObject = incref(val); } + static PyObject* lockType = staticPythonInstance("typed_python.internals", "lockType"); + + if (val == lockType) { + mKind = Kind::NamedPyObject; + mName = "lockType"; + mModuleName = "typed_python.internals"; + return; + } + static PyObject* environType = staticPythonInstance("os", "_Environ"); if (val->ob_type == (PyTypeObject*)environType) { diff --git a/typed_python/compiler/native_compiler/native_ast.py b/typed_python/compiler/native_compiler/native_ast.py index 4df2940f3..6f818c7ee 100644 --- a/typed_python/compiler/native_compiler/native_ast.py +++ b/typed_python/compiler/native_compiler/native_ast.py @@ -669,10 +669,6 @@ def expr_could_throw(self): Teardown = resolveForwardDefinedType(Teardown) CallTarget = resolveForwardDefinedType(CallTarget) -from typed_python._types import isForwardDefined -print(Expression.__typed_python_category__) -assert not isForwardDefined(Expression) - def ensureExpr(x): if isinstance(x, int): diff --git a/typed_python/internals.py b/typed_python/internals.py index bd7185f1f..e2e378f0d 100644 --- a/typed_python/internals.py +++ b/typed_python/internals.py @@ -44,6 +44,7 @@ class UndefinedBehaviorException(BaseException): # needed by the C api object = object type = type +lockType = type(threading.Lock()) class Final: From 7b7656cbbf817adfea82c676eca126395b2e4025 Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Mon, 7 Aug 2023 02:34:04 +0000 Subject: [PATCH 82/83] Working towards being able to internalize any kind of instance. Need to be able to formally separate the 'cache' layer of the snapshot from its definition. Need to move all PyObject in the Type layer to Instance. Need to amek sure that Instance internalization is sensible. What to do about weird things like held classes or pointers? --- todo.txt | 47 +------- typed_python/PyAlternativeInstance.cpp | 43 +++---- typed_python/PyClassInstance.cpp | 43 +++---- typed_python/PyInstance.cpp | 11 +- typed_python/PyObjGraphSnapshot.cpp | 5 +- typed_python/PyObjGraphSnapshot.hpp | 2 +- typed_python/PyObjSnapshot.cpp | 151 ++++++++++++++++++++++++- typed_python/PyObjSnapshot.hpp | 30 +++-- typed_python/PyObjSnapshotMaker.cpp | 5 +- typed_python/PyObjSnapshotMaker.hpp | 7 +- typed_python/PyPyObjSnapshot.cpp | 5 +- typed_python/PySetInstance.cpp | 38 ++++--- typed_python/Type.cpp | 1 + typed_python/Type.hpp | 1 - typed_python/ValueType.cpp | 12 ++ typed_python/ValueType.hpp | 6 +- typed_python/py_obj_snapshot_test.py | 15 ++- typed_python/type_construction_test.py | 38 +++---- 18 files changed, 287 insertions(+), 173 deletions(-) diff --git a/todo.txt b/todo.txt index b33d2a489..aa3c72534 100644 --- a/todo.txt +++ b/todo.txt @@ -220,46 +220,9 @@ how to do this: - - - - - - - - - - - - - - - -D = Forward(lambda: X) - -class C: - MyD = D - - -class D(Class): - MyC = C - - -How should we handle this case? - -At some level, both C and D are 'forward defined' because 'C' has a forward reference to D and vice versa. - -So, now we will have a new C and a new D, triggered at the moment we constructed 'D'. - -Of course, if we allowed 'C' to escape then we have a problem - - - -Of course, when we define C we wont' have - -at the module level, we shouldn't be replacing 'C', we should be updating it? - -how can we tell the difference between these two? - -Should 'C' have been considered 'Forward Defined'? +this is kind of working. But: +1. we need for everything in the core type graph to be 'Instance' +2. we need to not need to instantiate the PyTypeObject for a Type until the end +3. we should be able to do this by insisting that we wire the Instance of PythonObjectOfType + at the very end diff --git a/typed_python/PyAlternativeInstance.cpp b/typed_python/PyAlternativeInstance.cpp index dd4483577..a15ea3c19 100644 --- a/typed_python/PyAlternativeInstance.cpp +++ b/typed_python/PyAlternativeInstance.cpp @@ -795,33 +795,18 @@ PyObject* PyConcreteAlternativeInstance::altExit(PyObject* o, PyObject* args, Py // static PyMethodDef* PyConcreteAlternativeInstance::typeMethodsConcrete(Type* t) { - - // List of magic methods that are not attached to direct function pointers in PyTypeObject. - // These need to be defined by adding entries to PyTypeObject.tp_methods - // and we need to avoid adding them to PyTypeObject.tp_dict ourselves. - // Also, we only want to add the entry to tp_methods if they are explicitly defined. - const std::map special_magic_methods = { - {"__format__", (PyCFunction)altFormat}, - {"__bytes__", (PyCFunction)altBytes}, - {"__dir__", (PyCFunction)altDir}, - {"__reversed__", (PyCFunction)altReversed}, - {"__complex__", (PyCFunction)altComplex}, - {"__round__", (PyCFunction)altRound}, - {"__trunc__", (PyCFunction)altTrunc}, - {"__floor__", (PyCFunction)altFloor}, - {"__ceil__", (PyCFunction)altCeil}, - {"__enter__", (PyCFunction)altEnter}, - {"__exit__", (PyCFunction)altExit} - }; - - int cur = 0; - auto altMethods = ((ConcreteAlternative*)t)->getAlternative()->getMethods(); - PyMethodDef* ret = new PyMethodDef[special_magic_methods.size() + 1]; - for (auto m: special_magic_methods) { - if (altMethods.find(m.first) != altMethods.end()) { - ret[cur++] = {m.first, m.second, METH_VARARGS | METH_KEYWORDS, NULL}; - } - } - ret[cur] = {NULL, NULL}; - return ret; + return new PyMethodDef [12] { + {"__format__", (PyCFunction)altFormat, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__bytes__", (PyCFunction)altBytes, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__dir__", (PyCFunction)altDir, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__reversed__", (PyCFunction)altReversed, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__complex__", (PyCFunction)altComplex, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__round__", (PyCFunction)altRound, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__trunc__", (PyCFunction)altTrunc, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__floor__", (PyCFunction)altFloor, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__ceil__", (PyCFunction)altCeil, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__enter__", (PyCFunction)altEnter, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__exit__", (PyCFunction)altExit, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} + }; } diff --git a/typed_python/PyClassInstance.cpp b/typed_python/PyClassInstance.cpp index 8f1b914cb..ef5ea9777 100644 --- a/typed_python/PyClassInstance.cpp +++ b/typed_python/PyClassInstance.cpp @@ -1357,33 +1357,18 @@ PyObject* PyClassInstance::clsExitGeneric(PyObject* o, Type* t, instance_ptr dat // static PyMethodDef* PyClassInstance::typeMethodsConcrete(Type* t) { - - // List of magic methods that are not attached to direct function pointers in PyTypeObject. - // These need to be defined by adding entries to PyTypeObject.tp_methods - // and we need to avoid adding them to PyTypeObject.tp_dict ourselves. - // Also, we only want to add the entry to tp_methods if they are explicitly defined. - const std::map special_magic_methods = { - {"__format__", (PyCFunction)clsFormat}, - {"__bytes__", (PyCFunction)clsBytes}, - {"__dir__", (PyCFunction)clsDir}, - {"__reversed__", (PyCFunction)clsReversed}, - {"__complex__", (PyCFunction)clsComplex}, - {"__round__", (PyCFunction)clsRound}, - {"__trunc__", (PyCFunction)clsTrunc}, - {"__floor__", (PyCFunction)clsFloor}, - {"__ceil__", (PyCFunction)clsCeil}, - {"__enter__", (PyCFunction)clsEnter}, - {"__exit__", (PyCFunction)clsExit} - }; - - int cur = 0; - auto clsMethods = ((Class*)t)->getMemberFunctions(); - PyMethodDef* ret = new PyMethodDef[special_magic_methods.size() + 1]; - for (auto m: special_magic_methods) { - if (clsMethods.find(m.first) != clsMethods.end()) { - ret[cur++] = {m.first, m.second, METH_VARARGS | METH_KEYWORDS, NULL}; - } - } - ret[cur] = {NULL, NULL}; - return ret; + return new PyMethodDef [12] { + {"__format__", (PyCFunction)clsFormat, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__bytes__", (PyCFunction)clsBytes, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__dir__", (PyCFunction)clsDir, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__reversed__", (PyCFunction)clsReversed, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__complex__", (PyCFunction)clsComplex, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__round__", (PyCFunction)clsRound, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__trunc__", (PyCFunction)clsTrunc, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__floor__", (PyCFunction)clsFloor, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__ceil__", (PyCFunction)clsCeil, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__enter__", (PyCFunction)clsEnter, METH_VARARGS | METH_KEYWORDS, NULL}, + {"__exit__", (PyCFunction)clsExit, METH_VARARGS | METH_KEYWORDS, NULL}, + {NULL, NULL} + }; } diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 8b698f82b..608852859 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -1208,7 +1208,7 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { PyInstance::tp_iter : 0, // getiterfunc tp_iter; .tp_iternext = PyInstance::tp_iternext, // iternextfunc - .tp_methods = 0, // struct PyMethodDef* + .tp_methods = typeMethods(inType), // struct PyMethodDef* .tp_members = 0, // struct PyMemberDef* .tp_getset = 0, // struct PyGetSetDef* .tp_base = 0, // struct _typeobject* @@ -1260,6 +1260,9 @@ PyTypeObject* PyInstance::typeObjInternal(Type* inType) { ((PyObject*)&types[inType]->typeObj)->ob_type = incref(classMetaclass); } + PyType_Ready((PyTypeObject*)types[inType]); + inType->markTypeObjReady(); + if (!inType->isActivelyBeingDeserialized()) { finalizePyTypeObjectPhase1(inType, (PyTypeObject*)types[inType]); finalizePyTypeObjectPhase2(inType, (PyTypeObject*)types[inType]); @@ -1539,11 +1542,7 @@ bool PyInstance::typeCanBeSubclassed(Type* t) { // static void PyInstance::finalizePyTypeObjectPhase1(Type* inType, PyTypeObject* pyType) { - // update the type methods - pyType->tp_methods = typeMethods(inType); - - PyType_Ready(pyType); - inType->markTypeObjReady(); + //pyType->tp_name = (new std::string(inType->nameWithModule()))->c_str(); } void PyInstance::finalizePyTypeObjectPhase2(Type* inType, PyTypeObject* pyType) { diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index 8794553b0..b9ca990fe 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -17,7 +17,7 @@ #include "PyObjGraphSnapshot.hpp" #include "PyObjSnapshotGroupSearch.hpp" -PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBak) : +PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBack, bool linkToInternal) : mGroupsConsumed(0), mGroupSearch(this) { @@ -34,7 +34,8 @@ PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBak) : typeMapCache, instanceCache, this, - linkBak + linkBack, + linkToInternal ); snapMaker.internalize(root); diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index abf47811c..726cdeaaf 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -45,7 +45,7 @@ class PyObjGraphSnapshot { { } - PyObjGraphSnapshot(Type* root, bool linkBackToOriginal=true); + PyObjGraphSnapshot(Type* root, bool linkBackToOriginal=true, bool linkToInternal=true); ~PyObjGraphSnapshot() { for (auto oPtr: mObjects) { diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index 49debba44..c62755867 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -41,6 +41,57 @@ std::string PyObjSnapshot::kindAsString() const { if (mKind == Kind::Instance) { return "Instance"; } + if (mKind == Kind::ValueInstance) { + return "ValueInstance"; + } + if (mKind == Kind::FunctionInstance) { + return "FunctionInstance"; + } + if (mKind == Kind::ClassInstance) { + return "ClassInstance"; + } + if (mKind == Kind::TupleInstance) { + return "TupleInstance"; + } + if (mKind == Kind::NamedTupleInstance) { + return "NamedTupleInstance"; + } + if (mKind == Kind::TupleOfInstance) { + return "TupleOfInstance"; + } + if (mKind == Kind::ListOfInstance) { + return "ListOfInstance"; + } + if (mKind == Kind::SetInstance) { + return "SetInstance"; + } + if (mKind == Kind::DictInstance) { + return "DictInstance"; + } + if (mKind == Kind::ConstDictInstance) { + return "ConstDictInstance"; + } + if (mKind == Kind::PointerToInstance) { + return "PointerToInstance"; + } + if (mKind == Kind::RefToInstance) { + return "RefToInstance"; + } + if (mKind == Kind::AlternativeInstance) { + return "AlternativeInstance"; + } + if (mKind == Kind::ConcreteAlternativeInstance) { + return "ConcreteAlternativeInstance"; + } + if (mKind == Kind::AlternativeMatcherInstance) { + return "AlternativeMatcherInstance"; + } + if (mKind == Kind::BoundMethodInstance) { + return "BoundMethodInstance"; + } + if (mKind == Kind::TypedCellInstance) { + return "TypedCellInstance"; + } if (mKind == Kind::PrimitiveType) { return "PrimitiveType"; } @@ -490,8 +541,6 @@ void PyObjSnapshot::becomeInternalizedOf( throw std::runtime_error("Type of category " + t->getTypeCategoryString() + " can't be snapshotted"); } - - void PyObjSnapshot::becomeInternalizedOf( PyObject* val, PyObjSnapshotMaker& maker @@ -901,8 +950,102 @@ void PyObjSnapshot::becomeInternalizedOf( InstanceRef val, PyObjSnapshotMaker& maker ) { - mKind = Kind::Instance; - mInstance = val; + if (val.type()->isFunction()) { + mKind = Kind::FunctionInstance; + mNamedElements["type"] = maker.internalize(val.type()); + mNamedElements["closure"] = maker.internalize( + InstanceRef(val.data(), ((Function*)val.type())->getClosureType()) + ); + return; + } + + if (val.type()->isClass()) { + mKind = Kind::ClassInstance; + mNamedElements["type"] = maker.internalize(val.type()); + + // instances are supposed to hold actual classes. If we have a ListOf(Base) + // we expect that the "instance objects" representing a child of 'base' actually + // know that they are 'child' + Class* actual = Class::actualTypeForLayout(val.data()); + if (actual != val.type()) { + throw std::runtime_error( + "PyObjSnapshot expected instance of " + val.type()->name() + " to not be " + + "a subclass" + ); + } + + for (long i = 0; i < actual->getMembers().size(); i++) { + if (actual->checkInitializationFlag(val.data(), i)) { + mNamedElements["member" + format(i)] = maker.internalize( + InstanceRef( + actual->eltPtr(val.data(), i), + actual->getMembers()[i].getType() + ) + ); + } + } + + return; + } + + if (val.type()->isTupleOf() || val.type()->isListOf()) { + if (val.type()->isTupleOf()) { + mKind = Kind::TupleOfInstance; + } else { + mKind = Kind::ListOfInstance; + } + + mNamedElements["type"] = maker.internalize(val.type()); + TupleOrListOfType* nt = (TupleOrListOfType*)val.type(); + + for (long i = 0; i < nt->count(val.data()); i++) { + mElements.push_back( + maker.internalize( + InstanceRef( + nt->eltPtr(val.data(), i), + nt->getEltType() + ) + ) + ); + } + + return; + } + + if (val.type()->isNamedTuple() || val.type()->isTuple()) { + if (val.type()->isNamedTuple()) { + mKind = Kind::NamedTupleInstance; + } else { + mKind = Kind::TupleInstance; + } + + mNamedElements["type"] = maker.internalize(val.type()); + CompositeType* nt = (CompositeType*)val.type(); + + for (long i = 0; i < nt->getTypes().size(); i++) { + mElements.push_back( + maker.internalize( + InstanceRef( + nt->eltPtr(val.data(), i), + nt->getTypes()[i] + ) + ) + ); + } + + return; + } + + if (val.type()->isNone() || val.type()->isBytes() || val.type()->isRegister()) { + mKind = Kind::Instance; + mInstance = val; + return; + } + + throw std::runtime_error( + "Can't make a PyObjSnapshot out of an instance of type " + + mInstance.type()->name() + ); } void PyObjSnapshot::becomeInternalizedOf( diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 8df306961..8eead9d1f 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -59,14 +59,32 @@ class PyObjSnapshot { Uninitialized = 0, // a string held in mStringObject String, + // an instance of a FunctionType + FunctionInstance, + ValueInstance, + ClassInstance, + TupleInstance, + NamedTupleInstance, + TupleOfInstance, + ListOfInstance, + SetInstance, + DictInstance, + ConstDictInstance, + PointerToInstance, + RefToInstance, + AlternativeInstance, + ConcreteAlternativeInstance, + AlternativeMatcherInstance, + BoundMethodInstance, + TypedCellInstance, + // we're pointing into a TP leaf instance (a register type, none, or bytes) + PrimitiveInstance, // we're pointing to a "canonical" python object that's visible from a module // and that we don't want to look inside of. mName will contain the // name, and mModuleName will contain the module. We will assume that this object // is the same across program invocations (its a C function and we can't // look inside of it) NamedPyObject, - // we're pointing into a TP leaf instance (a register type, an int, bytes, etc.) - Instance, // we're a primitive type (like int) PrimitiveType, // a TP ListOf type. The element type will be in element_type @@ -187,10 +205,6 @@ class PyObjSnapshot { return mKind == Kind::String; } - bool isInstance() const { - return mKind == Kind::Instance; - } - std::string kindAsString() const; static std::string dictGetStringOrEmpty(PyObject* dict, const char* name) { @@ -430,9 +444,7 @@ class PyObjSnapshot { } bool needsHydration() const { - // currently, instances are 'out of band'. Eventually they'll be just like the - // other content. - if (mKind == Kind::Instance) { + if (mKind == Kind::PrimitiveInstance) { return false; } diff --git a/typed_python/PyObjSnapshotMaker.cpp b/typed_python/PyObjSnapshotMaker.cpp index f48d11b16..b66f71cc6 100644 --- a/typed_python/PyObjSnapshotMaker.cpp +++ b/typed_python/PyObjSnapshotMaker.cpp @@ -5,7 +5,6 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(const std::string& def) { PyObjSnapshot* res = new PyObjSnapshot(mGraph); - res->becomeInternalizedOf(def, *this); return res; @@ -177,6 +176,10 @@ PyObjSnapshot* PyObjSnapshotMaker::internalize(PyObject* val) /* static */ PyObjSnapshot* PyObjSnapshotMaker::internalize(Type* val) { + if (mLinkToInternal && val->getSnapshot()) { + return val->getSnapshot(); + } + auto it = mTypeMapCache.find(val); if (it != mTypeMapCache.end()) { diff --git a/typed_python/PyObjSnapshotMaker.hpp b/typed_python/PyObjSnapshotMaker.hpp index 3628ceedf..6a751c004 100644 --- a/typed_python/PyObjSnapshotMaker.hpp +++ b/typed_python/PyObjSnapshotMaker.hpp @@ -34,13 +34,15 @@ class PyObjSnapshotMaker { std::unordered_map& inTypeMapCache, std::unordered_map& inInstanceCache, PyObjGraphSnapshot* inGraph, - bool inLinkBackToOriginalObject + bool inLinkBackToOriginalObject, + bool linkToInternal ) : mObjMapCache(inObjMapCache), mTypeMapCache(inTypeMapCache), mInstanceCache(inInstanceCache), mGraph(inGraph), - mLinkBackToOriginalObject(inLinkBackToOriginalObject) + mLinkBackToOriginalObject(inLinkBackToOriginalObject), + mLinkToInternal(linkToInternal) { } @@ -82,5 +84,6 @@ class PyObjSnapshotMaker { std::unordered_map& mInstanceCache; PyObjGraphSnapshot* mGraph; bool mLinkBackToOriginalObject; + bool mLinkToInternal; }; diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index 77217c358..39204a5de 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -50,7 +50,10 @@ PyObject* PyPyObjSnapshot::create(PyObject* self, PyObject* args, PyObject* kwar constantMapCacheType, constantMapCacheInst, graph, - linkBack + linkBack, + // don't link to the internal graph since we don't have + // that modeled correctly in the python wrappers yet. + false ); PyObjSnapshot* object = maker.internalize(instance); diff --git a/typed_python/PySetInstance.cpp b/typed_python/PySetInstance.cpp index 582180cdc..218a1e463 100644 --- a/typed_python/PySetInstance.cpp +++ b/typed_python/PySetInstance.cpp @@ -1403,22 +1403,24 @@ bool PySetInstance::pyValCouldBeOfTypeConcrete(modeled_type* type, PyObject* pyR } PyMethodDef* PySetInstance::typeMethodsConcrete(Type* t) { - return new PyMethodDef[18]{{"add", (PyCFunction)PySetInstance::setAdd, METH_VARARGS, setAdd_doc}, - {"pop", (PyCFunction)PySetInstance::setPop, METH_VARARGS, setPop_doc}, - {"discard", (PyCFunction)PySetInstance::setDiscard, METH_VARARGS, setDiscard_doc}, - {"remove", (PyCFunction)PySetInstance::setRemove, METH_VARARGS, setRemove_doc}, - {"clear", (PyCFunction)PySetInstance::setClear, METH_VARARGS, setClear_doc}, - {"copy", (PyCFunction)PySetInstance::setCopy, METH_VARARGS, setCopy_doc}, - {"union", (PyCFunction)PySetInstance::setUnion, METH_VARARGS, setUnion_doc}, - {"update", (PyCFunction)PySetInstance::setUpdate, METH_VARARGS, setUpdate_doc}, - {"intersection", (PyCFunction)PySetInstance::setIntersection, METH_VARARGS, setIntersection_doc}, - {"intersection_update", (PyCFunction)PySetInstance::setIntersectionUpdate, METH_VARARGS, setIntersectionUpdate_doc}, - {"difference", (PyCFunction)PySetInstance::setDifference, METH_VARARGS, setDifference_doc}, - {"difference_update", (PyCFunction)PySetInstance::setDifferenceUpdate, METH_VARARGS, setDifferenceUpdate_doc}, - {"symmetric_difference", (PyCFunction)PySetInstance::setSymmetricDifference, METH_VARARGS, setSymmetricDifference_doc}, - {"symmetric_difference_update", (PyCFunction)PySetInstance::setSymmetricDifferenceUpdate, METH_VARARGS, setSymmetricDifferenceUpdate_doc}, - {"issubset", (PyCFunction)PySetInstance::setIsSubset, METH_VARARGS, setIsSubset_doc}, - {"issuperset", (PyCFunction)PySetInstance::setIsSuperset, METH_VARARGS, setIsSuperset_doc}, - {"isdisjoint", (PyCFunction)PySetInstance::setIsDisjoint, METH_VARARGS, setIsDisjoint_doc}, - {NULL, NULL}}; + return new PyMethodDef[18]{ + {"add", (PyCFunction)PySetInstance::setAdd, METH_VARARGS, setAdd_doc}, + {"pop", (PyCFunction)PySetInstance::setPop, METH_VARARGS, setPop_doc}, + {"discard", (PyCFunction)PySetInstance::setDiscard, METH_VARARGS, setDiscard_doc}, + {"remove", (PyCFunction)PySetInstance::setRemove, METH_VARARGS, setRemove_doc}, + {"clear", (PyCFunction)PySetInstance::setClear, METH_VARARGS, setClear_doc}, + {"copy", (PyCFunction)PySetInstance::setCopy, METH_VARARGS, setCopy_doc}, + {"union", (PyCFunction)PySetInstance::setUnion, METH_VARARGS, setUnion_doc}, + {"update", (PyCFunction)PySetInstance::setUpdate, METH_VARARGS, setUpdate_doc}, + {"intersection", (PyCFunction)PySetInstance::setIntersection, METH_VARARGS, setIntersection_doc}, + {"intersection_update", (PyCFunction)PySetInstance::setIntersectionUpdate, METH_VARARGS, setIntersectionUpdate_doc}, + {"difference", (PyCFunction)PySetInstance::setDifference, METH_VARARGS, setDifference_doc}, + {"difference_update", (PyCFunction)PySetInstance::setDifferenceUpdate, METH_VARARGS, setDifferenceUpdate_doc}, + {"symmetric_difference", (PyCFunction)PySetInstance::setSymmetricDifference, METH_VARARGS, setSymmetricDifference_doc}, + {"symmetric_difference_update", (PyCFunction)PySetInstance::setSymmetricDifferenceUpdate, METH_VARARGS, setSymmetricDifferenceUpdate_doc}, + {"issubset", (PyCFunction)PySetInstance::setIsSubset, METH_VARARGS, setIsSubset_doc}, + {"issuperset", (PyCFunction)PySetInstance::setIsSuperset, METH_VARARGS, setIsSuperset_doc}, + {"isdisjoint", (PyCFunction)PySetInstance::setIsDisjoint, METH_VARARGS, setIsDisjoint_doc}, + {NULL, NULL} + }; } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 9e38d9d70..337faa024 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -269,6 +269,7 @@ void reachableUnresolvedTypes(Type* root, std::set& outTypes) { typeMapCache, instanceCache, &graph, + true, true ); diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index b3ecfd1b1..e8db84dc9 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -604,7 +604,6 @@ class Type { case catSubclassOf: return f(*(SubclassOfType*)this); default: - asm("int3"); throw std::runtime_error("Invalid type found: " + format((int)m_typeCategory)); } } diff --git a/typed_python/ValueType.cpp b/typed_python/ValueType.cpp index 57bb9d666..20901c089 100644 --- a/typed_python/ValueType.cpp +++ b/typed_python/ValueType.cpp @@ -140,6 +140,18 @@ void Value::updateInternalTypePointersConcrete( } } +void Value::initializeDuringDeserialization(Instance i) { + mInstance = i; + + mValueAsPyobj = PyInstance::extractPythonObject(mInstance); + + if (!mValueAsPyobj) { + PyErr_Clear(); + throw std::runtime_error("Failed to convert an instance of type " + mInstance.type()->name() + " to a PyObject!"); + } +} + + void Value::initializeFromConcrete(Type* forwardDefinitionOfSelf) { mInstance = ((Value*)forwardDefinitionOfSelf)->mInstance.clone(); mValueAsPyobj = PyInstance::extractPythonObject(mInstance); diff --git a/typed_python/ValueType.hpp b/typed_python/ValueType.hpp index dd322dded..01b4eb623 100644 --- a/typed_python/ValueType.hpp +++ b/typed_python/ValueType.hpp @@ -109,10 +109,8 @@ class Value : public Type { static Type* Make(Instance i); - void initializeDuringDeserialization(Instance i) { - mInstance = i; - } - + void initializeDuringDeserialization(Instance i); + void postInitializeConcrete() {} Type* cloneForForwardResolutionConcrete() { diff --git a/typed_python/py_obj_snapshot_test.py b/typed_python/py_obj_snapshot_test.py index 6ca0a6ebb..7ec3e25aa 100644 --- a/typed_python/py_obj_snapshot_test.py +++ b/typed_python/py_obj_snapshot_test.py @@ -1,6 +1,6 @@ import numpy -from typed_python import ListOf, Alternative, Forward +from typed_python import ListOf, Alternative, Forward, Function from typed_python._types import PyObjSnapshot, PyObjGraphSnapshot, _enableTypeAutoresolution @@ -292,3 +292,16 @@ def test_alternative_methods(): assert snap.shaHash == snap2.shaHash assert snap.pyobj.f.__name__ == 'f' + + +def test_snapshot_of_function(): + closureVar = 10 + + @Function + def f(): + return closureVar + + snap = PyObjSnapshot.create(type(f), False) + snap2 = PyObjSnapshot.create(snap.pyobj, False) + + assert snap.shaHash == snap2.shaHash diff --git a/typed_python/type_construction_test.py b/typed_python/type_construction_test.py index 77eb2be54..48ab4ec87 100644 --- a/typed_python/type_construction_test.py +++ b/typed_python/type_construction_test.py @@ -38,6 +38,18 @@ def f(x: int): def g(): return f(2) + print(g.ClosureType) + print(g.overloads[0]) + + from typed_python import _types + g2 = _types.prepareArgumentToBePassedToCompiler(g) + print("g2: ") + print(g2.overloads[0].globals) + print(g2.overloads[0].closureVarLookups) + print(g2.overloads[0].closureVarLookups['f'][0]) + print(g2.overloads[0].closureVars) + print() + assert g() == 3 @@ -64,25 +76,6 @@ def g(): assert m['h1'] == m['h2'] -def test_can_see_original_class_that_defined_a_forward(): - class C(Class): - x = Member(lambda: B) - - CFwd = (C,) - - assert isForwardDefined(C) - - # this is insufficient to trigger an autoresolve because - # we didn't create a new Type instance. - B = int - - C = resolveForwardDefinedType(C) - - assert not isForwardDefined(C) - - assert forwardDefinitionsFor(C) == [CFwd[0]] - - def test_forward_class_exposes_functions(): class C(Class): x = Member(lambda: B) @@ -137,7 +130,7 @@ def f(self): return C - assert CofX(1).f.overloads[0].globals['x'].kind == 'Constant' + assert CofX(1).f.overloads[0].globals['x'].kind == 'GlobalInCell' assert CofX(1).f.overloads[0].globals['x'].getValue() is 1 assert CofX(2).f.overloads[0].globals['x'].getValue() is 2 @@ -172,9 +165,8 @@ def test_type_looks_resolvable_alternative(): B = resolveForwardDefinedType(B) bGlobal = A.f.overloads[0].globals['B'] - assert bGlobal.kind == "Constant" - assert bGlobal.constant.kind == "PrimitiveType" - assert bGlobal.constant.type is B + assert bGlobal.kind == "GlobalInCell" + assert bGlobal.getValue() is B bFwdGlobal = AFwd[0].f.overloads[0].globals['B'] assert bFwdGlobal.kind == "GlobalInCell" From b23ae88165f1a58f01e18cb0e2c69dd07df2a81c Mon Sep 17 00:00:00 2001 From: Braxton Mckee Date: Fri, 11 Aug 2023 23:14:58 +0000 Subject: [PATCH 83/83] more of trying to get the graph to work correctly --- typed_python/AlternativeType.hpp | 26 +- typed_python/ConcreteAlternativeType.hpp | 15 +- typed_python/ForwardType.hpp | 1 - typed_python/FunctionType.hpp | 38 +- typed_python/PyInstance.cpp | 3 + typed_python/PyObjGraphSnapshot.cpp | 17 +- typed_python/PyObjGraphSnapshot.hpp | 2 + typed_python/PyObjRehydrator.cpp | 408 +++++++++++------- typed_python/PyObjRehydrator.hpp | 7 +- typed_python/PyObjSnapshot.cpp | 198 ++++++++- typed_python/PyObjSnapshot.hpp | 32 +- typed_python/PyObjSnapshotGroupSearch.hpp | 8 +- typed_python/PyPyObjSnapshot.cpp | 4 - ...onSerializationContext_deserialization.cpp | 2 +- typed_python/Type.cpp | 4 + typed_python/Type.hpp | 36 +- typed_python/TypeStack.hpp | 28 +- typed_python/_runtime.cpp | 2 +- 18 files changed, 548 insertions(+), 283 deletions(-) diff --git a/typed_python/AlternativeType.hpp b/typed_python/AlternativeType.hpp index 862d93240..bb632175a 100644 --- a/typed_python/AlternativeType.hpp +++ b/typed_python/AlternativeType.hpp @@ -109,7 +109,7 @@ class Alternative : public Type { m_is_forward_defined = true; m_name = name; - m_moduleName = moduleName; + m_module_name = moduleName; if (m_subtypes.size() > 255) { throw std::runtime_error("Can't have an alternative with more than 255 subelements"); @@ -128,7 +128,7 @@ class Alternative : public Type { const std::vector& subtypes_concrete ) { m_name = name; - m_moduleName = moduleName; + m_module_name = moduleName; m_methods = methods; m_subtypes = types; m_subtypes_concrete = subtypes_concrete; @@ -142,7 +142,7 @@ class Alternative : public Type { Alternative* selfT = (Alternative*)forwardDefinitionOfSelf; m_name = selfT->m_name; - m_moduleName = selfT->m_moduleName; + m_module_name = selfT->m_module_name; m_hasGetAttributeMagicMethod = selfT->m_hasGetAttributeMagicMethod; m_methods = selfT->m_methods; m_subtypes = selfT->m_subtypes; @@ -209,22 +209,6 @@ class Alternative : public Type { } } - std::string nameWithModuleConcrete() { - if (m_moduleName.size() == 0) { - return m_name; - } - - return m_moduleName + "." + m_name; - } - - std::string moduleNameConcrete() { - if (m_moduleName.size() == 0) { - return "builtins"; - } - - return m_moduleName; - } - bool hasGetAttributeMagicMethod() const { return m_hasGetAttributeMagicMethod; } @@ -401,7 +385,7 @@ class Alternative : public Type { ); Alternative* renamed(std::string newName) { - return Make(newName, m_moduleName, m_subtypes, m_methods); + return Make(newName, m_module_name, m_subtypes, m_methods); } const std::vector >& subtypes() const { @@ -432,8 +416,6 @@ class Alternative : public Type { void initializeConcreteSubclasses(); //name of the module in which this Alternative was defined. - std::string m_moduleName; - bool m_all_alternatives_empty; int m_default_construction_ix; diff --git a/typed_python/ConcreteAlternativeType.hpp b/typed_python/ConcreteAlternativeType.hpp index 0d59577a1..10ca91da8 100644 --- a/typed_python/ConcreteAlternativeType.hpp +++ b/typed_python/ConcreteAlternativeType.hpp @@ -37,6 +37,7 @@ class ConcreteAlternative : public Type { { m_is_forward_defined = true; m_name = m_alternative->name() + "." + m_alternative->subtypes()[m_which].first; + m_module_name = m_alternative->moduleName(); } void initializeDuringDeserialization(int64_t which, Alternative* base) { @@ -46,7 +47,7 @@ class ConcreteAlternative : public Type { } const char* docConcrete() { - return m_alternative->doc(); + return Alternative_doc; } std::string computeRecursiveNameConcrete(TypeStack& typeStack) { @@ -93,18 +94,6 @@ class ConcreteAlternative : public Type { m_is_default_constructible = is_default_constructible; } - std::string nameWithModuleConcrete() { - if (m_alternative->moduleName().size() == 0) { - return m_name; - } - - return m_alternative->moduleName() + "." + m_name; - } - - std::string moduleNameConcrete() { - return m_alternative->moduleName(); - } - void deepcopyConcrete( instance_ptr dest, instance_ptr src, diff --git a/typed_python/ForwardType.hpp b/typed_python/ForwardType.hpp index 03e7e2e01..124c43fd7 100644 --- a/typed_python/ForwardType.hpp +++ b/typed_python/ForwardType.hpp @@ -137,7 +137,6 @@ class Forward : public Type { void setName(std::string newName) { m_name = newName; - m_stripped_name = ""; } void setCellOrDict(PyObject* newCellOrDict) { diff --git a/typed_python/FunctionType.hpp b/typed_python/FunctionType.hpp index dd47ddfcf..7741baeaa 100644 --- a/typed_python/FunctionType.hpp +++ b/typed_python/FunctionType.hpp @@ -58,11 +58,11 @@ class Function : public Type { mIsNocompile(isNocompile), mRootName(inName), mQualname(qualname), - mModulename(moduleName), mClosureType(closureType) { m_is_forward_defined = true; m_name = mRootName; + m_module_name = moduleName; } void initializeDuringDeserialization( @@ -76,25 +76,13 @@ class Function : public Type { ) { mRootName = inName; mQualname = qualname; - mModulename = moduleName; + m_module_name = moduleName; mOverloads = overloads; mClosureType = closureType; mIsEntrypoint = isEntrypoint; mIsNocompile = isNocompile; } - std::string moduleNameConcrete() { - if (mModulename.size() == 0) { - return "builtins"; - } - - return mModulename; - } - - std::string nameWithModuleConcrete() { - return moduleNameConcrete() + "." + (mQualname.size() ? mQualname : mRootName); - } - // does this function have any of its globals that are not // resolved to actual values. if 'insistForwardsResolved' then we // return 'true' if the symbol resolves to a forward defined @@ -151,7 +139,7 @@ class Function : public Type { mIsNocompile = fwdFunc->mIsNocompile; mRootName = fwdFunc->mRootName; mQualname = fwdFunc->mQualname; - mModulename = fwdFunc->mModulename; + m_module_name = fwdFunc->m_module_name; } Type* cloneForForwardResolutionConcrete() { @@ -253,7 +241,7 @@ class Function : public Type { v.visitName(m_name); v.visitName(mRootName); v.visitName(mQualname); - v.visitName(mModulename); + v.visitName(m_module_name); v.visitTopo(mClosureType); @@ -297,7 +285,7 @@ class Function : public Type { return Function::Make( f1->mRootName, f1->mQualname, - f1->mModulename, + f1->m_module_name, overloads, Tuple::Make(types), f1->isEntrypoint() || f2->isEntrypoint(), @@ -448,7 +436,7 @@ class Function : public Type { Function* withEntrypoint(bool isEntrypoint) const { Function* f = Function::Make( - mRootName, mQualname, mModulename, mOverloads, mClosureType, isEntrypoint, mIsNocompile + mRootName, mQualname, m_module_name, mOverloads, mClosureType, isEntrypoint, mIsNocompile ); if (f->isForwardDefined() && !isForwardDefined()) { @@ -459,7 +447,7 @@ class Function : public Type { Function* withNocompile(bool isNocompile) const { Function* f = Function::Make( - mRootName, mQualname, mModulename, mOverloads, mClosureType, mIsEntrypoint, isNocompile + mRootName, mQualname, m_module_name, mOverloads, mClosureType, mIsEntrypoint, isNocompile ); if (f->isForwardDefined() && !isForwardDefined()) { return (Function*)f->forwardResolvesTo(); @@ -529,11 +517,11 @@ class Function : public Type { // } Function* replaceClosure(Type* closureType) const { - return Function::Make(mRootName, mQualname, mModulename, mOverloads, closureType, mIsEntrypoint, mIsNocompile); + return Function::Make(mRootName, mQualname, m_module_name, mOverloads, closureType, mIsEntrypoint, mIsNocompile); } Function* replaceOverloads(const std::vector& overloads) const { - return Function::Make(mRootName, mQualname, mModulename, overloads, mClosureType, mIsEntrypoint, mIsNocompile); + return Function::Make(mRootName, mQualname, m_module_name, overloads, mClosureType, mIsEntrypoint, mIsNocompile); } Function* replaceOverloadVariableBindings(long index, const std::map& bindings) { @@ -544,7 +532,7 @@ class Function : public Type { overloads[index] = overloads[index].withClosureBindings(bindings); - return Function::Make(mRootName, mQualname, mModulename, overloads, mClosureType, mIsEntrypoint, mIsNocompile); + return Function::Make(mRootName, mQualname, m_module_name, overloads, mClosureType, mIsEntrypoint, mIsNocompile); } std::string qualname() const { @@ -555,10 +543,6 @@ class Function : public Type { return m_name; } - std::string moduleName() const { - return mModulename; - } - private: std::vector mOverloads; @@ -568,5 +552,5 @@ class Function : public Type { bool mIsNocompile; - std::string mRootName, mQualname, mModulename; + std::string mRootName, mQualname; }; diff --git a/typed_python/PyInstance.cpp b/typed_python/PyInstance.cpp index 608852859..daaf304dd 100644 --- a/typed_python/PyInstance.cpp +++ b/typed_python/PyInstance.cpp @@ -704,6 +704,9 @@ int PyInstance::sq_ass_item_concrete(Py_ssize_t ix, PyObject* v) { PyTypeObject* PyInstance::typeObj(Type* inType) { if (!inType->getTypeRep()) { inType->setTypeRep(typeObjInternal(inType)); + if (!typeObjInternal(inType)) { + throw std::runtime_error("Somehow, typeObjInternal didn't produce a type."); + } } return inType->getTypeRep(); diff --git a/typed_python/PyObjGraphSnapshot.cpp b/typed_python/PyObjGraphSnapshot.cpp index b9ca990fe..f15b5ed28 100644 --- a/typed_python/PyObjGraphSnapshot.cpp +++ b/typed_python/PyObjGraphSnapshot.cpp @@ -34,7 +34,7 @@ PyObjGraphSnapshot::PyObjGraphSnapshot(Type* root, bool linkBack, bool linkToInt typeMapCache, instanceCache, this, - linkBack, + linkBack, linkToInternal ); @@ -102,6 +102,19 @@ void PyObjGraphSnapshot::resolveForwards() { } } +void PyObjGraphSnapshot::recomputeNames() { + for (auto o: mObjects) { + mGroupSearch.add(o, false); + } + + for (auto& group: mGroupSearch.getGroups()) { + for (auto elt: *group) { + if (elt->willBeATpType()) { + elt->recomputeRecursiveName(*group); + } + } + } +} void PyObjGraphSnapshot::internalize() { internal().internalize(*this, false); @@ -161,7 +174,7 @@ PyObjSnapshot* PyObjGraphSnapshot::createSkeleton(ShaHash h) { template ShaHash computeHashFor(PyObjSnapshot* snap, const compute_type& compute) { - if (snap->getKind() == PyObjSnapshot::Kind::Instance) { + if (snap->getKind() == PyObjSnapshot::Kind::PrimitiveInstance) { if (snap->getInstance().type()->isPOD()) { return ShaHash(int(snap->getKind()), int(snap->getInstance().type()->getTypeCategory())) + ShaHash::SHA1(snap->getInstance().data(), snap->getInstance().type()->bytecount()); diff --git a/typed_python/PyObjGraphSnapshot.hpp b/typed_python/PyObjGraphSnapshot.hpp index 726cdeaaf..e8713c196 100644 --- a/typed_python/PyObjGraphSnapshot.hpp +++ b/typed_python/PyObjGraphSnapshot.hpp @@ -66,6 +66,8 @@ class PyObjGraphSnapshot { // internalize a graph into ourselves void internalize(PyObjGraphSnapshot& graph, bool markInternalized); + void recomputeNames(); + // get the "internal" graph snapshot, which is responsible for holding all the objects // that are actually interned inside the system. static PyObjGraphSnapshot& internal() { diff --git a/typed_python/PyObjRehydrator.cpp b/typed_python/PyObjRehydrator.cpp index dc1d4d649..7937c4b33 100644 --- a/typed_python/PyObjRehydrator.cpp +++ b/typed_python/PyObjRehydrator.cpp @@ -29,10 +29,127 @@ void PyObjRehydrator::start(PyObjSnapshot* snapshot) { }; visit(snapshot); +} + +void PyObjRehydrator::buildPyobjForTpType(PyObjSnapshot* s) { + if (!s->mType) { + throw std::runtime_error("Somehow this Snapshot doesn't have a Type."); + } + if (s->mPyObject) { + return; + } + + // these objects don't actually build a python representation for the type - we always + // use the actual type + if (s->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + return; + } + + s->mPyObject = incref((PyObject*)PyInstance::typeObj(s->mType)); + + if (!s->mPyObject) { + throw std::runtime_error("Somehow we don't have a PyObject for our type."); + } +} + + +void PyObjRehydrator::rehydrateAll() { + // first, do a pass building the skeletons for all of our objects. + // after this, there will be a Type* allocated for every new Type object with the + // correct name, which is enough to construct the PyTypeObject for it. + for (auto s: mSnapshots) { + if (s->willBeATpType()) { + buildTypeSkeleton(s); + } + } + + // now that every TP type knows its name, we can instantiate it + for (auto s: mSnapshots) { + if (s->willBeATpType()) { + buildPyobjForTpType(s); + } + } + + // then actually initializethe type objects. these will build PyObjects as needed + for (auto s: mSnapshots) { + if (s->willBeATpType()) { + rehydrateTpType(s); + } + } for (auto s: mSnapshots) { rehydrate(s); } + + for (auto n: mSnapshots) { + finalizeRehydration(n); + } + + std::vector types; + for (auto o: mSnapshots) { + if (o->mType) { + types.push_back(o->mType); + } + } + + std::sort(types.begin(), types.end(), [&](Type* l, Type* r) { + if (l->typeLevel() < r->typeLevel()) { + return true; + } + if (l->typeLevel() > r->typeLevel()) { + return false; + } + return l < r; + }); + + bool anyUpdated = true; + size_t passCt = 0; + while (anyUpdated) { + anyUpdated = false; + for (auto t: types) { + if (t->postInitialize()) { + anyUpdated = true; + } + } + passCt += 1; + + // we can run this algorithm until all type sizes have stabilized. Conceivably we + // could introduce an error that would cause this to not converge - this should + // detect that. + if (passCt > types.size() * 2 + 10) { + throw std::runtime_error("Type size graph is not stabilizing."); + } + } + + // let each type update any internal caches it might need before it gets instantiated + for (auto t: types) { + t->finalizeType(); + } + + for (auto t: types) { + t->typeFinishedBeingDeserializedPhase1(); + } + + // now that we've set function globals, we need to go over all the classes and + // held classes and make sure the versions of the function types they're actually + // using have the new definitions. Unfortunately, Overload objects are not pointers + // (maybe they should be?) and so when we merge Overloads from base/child classes + // they don't get their globals replaced with the appropriate values + for (auto& t: types) { + if (t && t->isHeldClass()) { + ((HeldClass*)t)->mergeOwnFunctionsIntoInheritanceTree(); + } + } + + for (auto t: types) { + t->typeFinishedBeingDeserializedPhase2(); + } + + for (auto obj: mSnapshots) { + if (obj->mType) { + obj->mPyObject = incref((PyObject*)PyInstance::typeObj(obj->mType)); + } + } } Type* PyObjRehydrator::typeFor(PyObjSnapshot* snapshot) { @@ -296,28 +413,107 @@ Type* PyObjRehydrator::getNamedElementType( return typeFor(it->second); } -void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { - if (snap->mType) { +void PyObjRehydrator::buildTypeSkeleton(PyObjSnapshot* snap) { + if (snap->mKind == PyObjSnapshot::Kind::PrimitiveType) { + // nothing to do return; } + if (snap->mType) { + throw std::runtime_error("Somehow this snapshot already has a Type?"); + } + if (snap->mKind == PyObjSnapshot::Kind::ListOfType) { snap->mType = new ListOfType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); + } else + if (snap->mKind == PyObjSnapshot::Kind::TupleOfType) { + snap->mType = new TupleOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::DictType) { + snap->mType = new DictType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ConstDictType) { + snap->mType = new ConstDictType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::SetType) { + snap->mType = new SetType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::PointerToType) { + snap->mType = new PointerTo(); + } else + if (snap->mKind == PyObjSnapshot::Kind::RefToType) { + snap->mType = new RefTo(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ValueType) { + snap->mType = new Value(); + } else + if (snap->mKind == PyObjSnapshot::Kind::OneOfType) { + snap->mType = new OneOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + snap->mType = new PythonObjectOfType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::TupleType) { + snap->mType = new Tuple(); + } else + if (snap->mKind == PyObjSnapshot::Kind::NamedTupleType) { + snap->mType = new NamedTuple(); + } else + if (snap->mKind == PyObjSnapshot::Kind::BoundMethodType) { + snap->mType = new BoundMethod(); + } else + if (snap->mKind == PyObjSnapshot::Kind::TypedCellType) { + snap->mType = new TypedCellType(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ForwardType) { + snap->mType = new Forward(snap->mName); + } else + if (snap->mKind == PyObjSnapshot::Kind::ConcreteAlternativeType) { + snap->mType = new ConcreteAlternative(); + } else + if (snap->mKind == PyObjSnapshot::Kind::AlternativeMatcherType) { + snap->mType = new AlternativeMatcher(); + } else + if (snap->mKind == PyObjSnapshot::Kind::AlternativeType) { + snap->mType = new Alternative(); + } else + if (snap->mKind == PyObjSnapshot::Kind::ClassType) { + snap->mType = new Class(); + } else + if (snap->mKind == PyObjSnapshot::Kind::HeldClassType) { + snap->mType = new HeldClass(); + } else + if (snap->mKind == PyObjSnapshot::Kind::FunctionType) { + snap->mType = new Function(); + } else { + throw std::runtime_error("Can't rehydrate a PyObjSnapshot of kind " + snap->kindAsString()); + } + + snap->mType->markActivelyBeingDeserialized( + snap->mNamedInts["type_is_forward"], + snap->mModuleName, + snap->mName + ); + + return; +} + +void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { + if (!snap->mType) { + return; + } + + if (snap->mKind == PyObjSnapshot::Kind::ListOfType) { ((ListOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); return; } if (snap->mKind == PyObjSnapshot::Kind::TupleOfType) { - snap->mType = new TupleOfType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((TupleOfType*)snap->mType)->initializeDuringDeserialization(getNamedElementType(snap, "element_type")); return; } if (snap->mKind == PyObjSnapshot::Kind::DictType) { - snap->mType = new DictType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((DictType*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "key_type"), getNamedElementType(snap, "value_type") @@ -326,8 +522,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::ConstDictType) { - snap->mType = new ConstDictType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((ConstDictType*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "key_type"), getNamedElementType(snap, "value_type") @@ -336,8 +530,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::SetType) { - snap->mType = new SetType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((SetType*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "key_type") ); @@ -345,8 +537,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::PointerToType) { - snap->mType = new PointerTo(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((PointerTo*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "element_type") ); @@ -354,8 +544,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::RefToType) { - snap->mType = new RefTo(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); ((RefTo*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "element_type") ); @@ -363,8 +551,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::ValueType) { - snap->mType = new Value(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); if (snap->mNamedElements.find("value_instance") == snap->mNamedElements.end()) { throw std::runtime_error("Corrupt PyObjSnapshot::Value"); } @@ -381,9 +567,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::OneOfType) { - snap->mType = new OneOfType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - std::vector types; for (auto s: snap->mElements) { types.push_back(typeFor(s)); @@ -395,9 +578,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { - snap->mType = new PythonObjectOfType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - PyObject* obType = getNamedElementPyobj(snap, "element_type", false); if (!PyType_Check(obType)) { @@ -409,9 +589,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::TupleType) { - snap->mType = new Tuple(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - std::vector types; for (auto s: snap->mElements) { types.push_back(typeFor(s)); @@ -423,9 +600,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::NamedTupleType) { - snap->mType = new NamedTuple(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - std::vector types; for (auto s: snap->mElements) { types.push_back(typeFor(s)); @@ -437,9 +611,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::BoundMethodType) { - snap->mType = new BoundMethod(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - if (snap->mNames.size() != 1) { throw std::runtime_error("Corrupt PyObjSnapshot::BoundMethod"); } @@ -453,9 +624,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::TypedCellType) { - snap->mType = new TypedCellType(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - ((TypedCellType*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "element_type") ); @@ -464,9 +632,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::ForwardType) { - snap->mType = new Forward(snap->mName); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - if (snap->mNamedElements.find("fwd_cell_or_dict") != snap->mNamedElements.end()) { ((Forward*)snap->mType)->setCellOrDict( getNamedElementPyobj(snap, "fwd_cell_or_dict") @@ -483,9 +648,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::ConcreteAlternativeType) { - snap->mType = new ConcreteAlternative(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - if (!getNamedElementType(snap, "alternative")->isAlternative()){ throw std::runtime_error("Corrupt PyObjSnapshot.Alternative"); } @@ -499,9 +661,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::AlternativeMatcherType) { - snap->mType = new AlternativeMatcher(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - ((AlternativeMatcher*)snap->mType)->initializeDuringDeserialization( getNamedElementType(snap, "alternative") ); @@ -510,9 +669,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::AlternativeType) { - snap->mType = new Alternative(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - std::vector > types; std::map methods; std::vector subtypesConcrete; @@ -547,9 +703,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::ClassType) { - snap->mType = new Class(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - Type* heldClass = getNamedElementType(snap, "held_class_type"); if (!heldClass->isHeldClass()) { @@ -564,9 +717,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::HeldClassType) { - snap->mType = new HeldClass(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - std::vector bases; std::vector members; std::map memberFunctions; @@ -626,9 +776,6 @@ void PyObjRehydrator::rehydrateTpType(PyObjSnapshot* snap) { } if (snap->mKind == PyObjSnapshot::Kind::FunctionType) { - snap->mType = new Function(); - snap->mType->markActivelyBeingDeserialized(snap->mNamedInts["type_is_forward"]); - std::string name = snap->mName; std::string qualname = snap->mQualname; std::string moduleName = snap->mModuleName; @@ -662,11 +809,6 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { ); } - if (obj->willBeATpType()) { - rehydrateTpType(obj); - return; - } - PyObjSnapshot::Kind kind = obj->mKind; PyObject*& pyObject(obj->mPyObject); @@ -677,15 +819,26 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { static PyObject* sysModuleModules = staticPythonInstance("sys", "modules"); + if (kind == PyObjSnapshot::Kind::PythonObjectOfTypeType) { + // PythonObjectOfType is special because its Type* doesn't have a builtin + // representation of the Type as a PythonTypeObject since we just use the real thing. + // this is similar to how we have a type OneOf(T1, T2,...) but never see an instance + // of it. + pyObject = getNamedElementPyobj(obj, "element_type", false); + } else if (kind == PyObjSnapshot::Kind::ArbitraryPyObject) { throw std::runtime_error("Corrupt PyObjSnapshot.ArbitraryPyObject: missing pyObject"); - } else if (kind == PyObjSnapshot::Kind::PrimitiveType) { + } else +if (kind == PyObjSnapshot::Kind::PrimitiveType) { pyObject = (PyObject*)PyInstance::typeObj(obj->mType); - } else if (kind == PyObjSnapshot::Kind::String) { + } else +if (kind == PyObjSnapshot::Kind::String) { pyObject = PyUnicode_FromString(obj->mStringValue.c_str()); - } else if (kind == PyObjSnapshot::Kind::Instance) { + } else +if (kind == PyObjSnapshot::Kind::PrimitiveInstance) { pyObject = PyInstance::extractPythonObject(obj->mInstance); - } else if (kind == PyObjSnapshot::Kind::NamedPyObject) { + } else +if (kind == PyObjSnapshot::Kind::NamedPyObject) { PyObjectStealer nameAsStr(PyUnicode_FromString(obj->mModuleName.c_str())); PyObjectStealer moduleObject( PyObject_GetItem(sysModuleModules, nameAsStr) @@ -705,15 +858,18 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { } pyObject = incref(inst); - } else if (kind == PyObjSnapshot::Kind::PyDict || kind == PyObjSnapshot::Kind::PyClassDict) { + } else +if (kind == PyObjSnapshot::Kind::PyDict || kind == PyObjSnapshot::Kind::PyClassDict) { pyObject = PyDict_New(); - } else if (kind == PyObjSnapshot::Kind::PyList) { + } else +if (kind == PyObjSnapshot::Kind::PyList) { pyObject = PyList_New(0); for (long k = 0; k < obj->mElements.size(); k++) { PyList_Append(pyObject, pyobjFor(obj->mElements[k])); } - } else if (kind == PyObjSnapshot::Kind::PyTuple) { + } else +if (kind == PyObjSnapshot::Kind::PyTuple) { pyObject = PyTuple_New(obj->mElements.size()); // first initialize it in case we throw somehow @@ -724,13 +880,15 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { for (long k = 0; k < obj->mElements.size(); k++) { PyTuple_SetItem(pyObject, k, incref(pyobjFor(obj->mElements[k]))); } - } else if (kind == PyObjSnapshot::Kind::PySet) { + } else +if (kind == PyObjSnapshot::Kind::PySet) { pyObject = PySet_New(nullptr); for (long k = 0; k < obj->mElements.size(); k++) { PySet_Add(pyObject, incref(pyobjFor(obj->mElements[k]))); } - } else if (kind == PyObjSnapshot::Kind::PyClass) { + } else +if (kind == PyObjSnapshot::Kind::PyClass) { PyObjectStealer argTup(PyTuple_New(3)); PyTuple_SetItem(argTup, 0, PyUnicode_FromString(obj->mName.c_str())); PyTuple_SetItem(argTup, 1, incref(getNamedElementPyobj(obj, "cls_bases"))); @@ -741,7 +899,8 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { if (!pyObject) { throw PythonExceptionSet(); } - } else if (kind == PyObjSnapshot::Kind::PyModule) { + } else +if (kind == PyObjSnapshot::Kind::PyModule) { PyObjectStealer nameAsStr(PyUnicode_FromString(obj->mName.c_str())); PyObjectStealer moduleObject( PyObject_GetItem(sysModuleModules, nameAsStr) @@ -754,14 +913,17 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { } pyObject = incref(moduleObject); - } else if (kind == PyObjSnapshot::Kind::PyModuleDict) { + } else +if (kind == PyObjSnapshot::Kind::PyModuleDict) { pyObject = PyObject_GenericGetDict(getNamedElementPyobj(obj, "module_dict_of"), nullptr); if (!pyObject) { throw PythonExceptionSet(); } - } else if (kind == PyObjSnapshot::Kind::PyCell) { + } else +if (kind == PyObjSnapshot::Kind::PyCell) { pyObject = PyCell_New(nullptr); - } else if (kind == PyObjSnapshot::Kind::PyObject) { + } else +if (kind == PyObjSnapshot::Kind::PyObject) { PyObject* t = getNamedElementPyobj(obj, "inst_type"); PyObject* d = getNamedElementPyobj(obj, "inst_dict"); @@ -776,7 +938,8 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { if (PyObject_GenericSetDict(pyObject, d, nullptr)) { throw PythonExceptionSet(); } - } else if (kind == PyObjSnapshot::Kind::PyFunction) { + } else +if (kind == PyObjSnapshot::Kind::PyFunction) { pyObject = PyFunction_New( getNamedElementPyobj(obj, "func_code"), getNamedElementPyobj(obj, "func_globals") @@ -826,7 +989,8 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { getNamedElementPyobj(obj, "func_name") ); } - } else if (kind == PyObjSnapshot::Kind::PyCodeObject) { + } else +if (kind == PyObjSnapshot::Kind::PyCodeObject) { #if PY_MINOR_VERSION < 8 pyObject = (PyObject*)PyCode_New( #else @@ -857,19 +1021,22 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { #endif ) ); - } else if (kind == PyObjSnapshot::Kind::PyStaticMethod) { + } else +if (kind == PyObjSnapshot::Kind::PyStaticMethod) { pyObject = PyStaticMethod_New(Py_None); JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)pyObject; decref(method->cm_callable); method->cm_callable = incref(getNamedElementPyobj(obj, "meth_func")); - } else if (kind == PyObjSnapshot::Kind::PyClassMethod) { + } else +if (kind == PyObjSnapshot::Kind::PyClassMethod) { pyObject = PyClassMethod_New(Py_None); JustLikeAClassOrStaticmethod* method = (JustLikeAClassOrStaticmethod*)pyObject; decref(method->cm_callable); method->cm_callable = incref(getNamedElementPyobj(obj, "meth_func")); - } else if (kind == PyObjSnapshot::Kind::PyProperty) { + } else +if (kind == PyObjSnapshot::Kind::PyProperty) { static PyObject* nones = PyTuple_Pack(3, Py_None, Py_None, Py_None); pyObject = PyObject_CallObject((PyObject*)&PyProperty_Type, nones); @@ -900,7 +1067,8 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { decref(dest->prop_name); dest->prop_name = incref(getNamedElementPyobj(obj, "prop_name", true)); #endif - } else if (kind == PyObjSnapshot::Kind::PyBoundMethod) { + } else +if (kind == PyObjSnapshot::Kind::PyBoundMethod) { pyObject = PyMethod_New(Py_None, Py_None); PyMethodObject* method = (PyMethodObject*)pyObject; decref(method->im_func); @@ -911,6 +1079,11 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { method->im_func = incref(getNamedElementPyobj(obj, "meth_func")); method->im_self = incref(getNamedElementPyobj(obj, "meth_self")); } else { + if (obj->willBeATpType()) { + asm("int3"); + throw std::runtime_error("Expected PyObjSnapshot of kind " + obj->kindAsString() + " to already have a pyobj."); + } + throw std::runtime_error( "Can't make a python object representation for a PyObjSnapshot of kind " + obj->kindAsString() @@ -918,81 +1091,6 @@ void PyObjRehydrator::rehydrate(PyObjSnapshot* obj) { } } -void PyObjRehydrator::finalize() { - for (auto n: mSnapshots) { - finalizeRehydration(n); - } - - for (auto n: mSnapshots) { - if (n->willBeATpType()) { - if (!n->mType) { - throw std::runtime_error( - "Snapshot " + n->toString() + " doesn't have a Type" - ); - } - } - } - - std::vector types; - for (auto o: mSnapshots) { - if (o->mType) { - types.push_back(o->mType); - } - } - - std::sort(types.begin(), types.end(), [&](Type* l, Type* r) { - if (l->typeLevel() < r->typeLevel()) { - return true; - } - if (l->typeLevel() > r->typeLevel()) { - return false; - } - return l < r; - }); - - for (auto t: types) { - t->recomputeName(); - } - - bool anyUpdated = true; - size_t passCt = 0; - while (anyUpdated) { - anyUpdated = false; - for (auto t: types) { - if (t->postInitialize()) { - anyUpdated = true; - } - } - passCt += 1; - - // we can run this algorithm until all type sizes have stabilized. Conceivably we - // could introduce an error that would cause this to not converge - this should - // detect that. - if (passCt > types.size() * 2 + 10) { - throw std::runtime_error("Type size graph is not stabilizing."); - } - } - - // let each type update any internal caches it might need before it gets instantiated - for (auto t: types) { - t->finalizeType(); - } - - for (auto t: types) { - t->typeFinishedBeingDeserializedPhase1(); - } - - for (auto t: types) { - t->typeFinishedBeingDeserializedPhase2(); - } - - for (auto obj: mSnapshots) { - if (obj->mType) { - obj->mPyObject = incref((PyObject*)PyInstance::typeObj(obj->mType)); - } - } -} - void PyObjRehydrator::finalizeRehydration(PyObjSnapshot* obj) { if (obj->mKind == PyObjSnapshot::Kind::PyClass) { if (obj->mNamedElements.find("cls_dict") == obj->mNamedElements.end()) { diff --git a/typed_python/PyObjRehydrator.hpp b/typed_python/PyObjRehydrator.hpp index a48e85a77..69ce0d701 100644 --- a/typed_python/PyObjRehydrator.hpp +++ b/typed_python/PyObjRehydrator.hpp @@ -30,7 +30,7 @@ class PyObjRehydrator { PyObject* pyobjFor(PyObjSnapshot* snapshot); - void finalize(); + void rehydrateAll(); PyObject* getNamedElementPyobj( PyObjSnapshot* snapshot, @@ -172,11 +172,14 @@ class PyObjRehydrator { } private: - void rehydrate(PyObjSnapshot* snapshot); + void buildTypeSkeleton(PyObjSnapshot* snap); + void rehydrateTpType(PyObjSnapshot* snap); + void buildPyobjForTpType(PyObjSnapshot* s); + void finalizeRehydration(PyObjSnapshot* snap); std::unordered_set mSnapshots; diff --git a/typed_python/PyObjSnapshot.cpp b/typed_python/PyObjSnapshot.cpp index c62755867..2c60d9637 100644 --- a/typed_python/PyObjSnapshot.cpp +++ b/typed_python/PyObjSnapshot.cpp @@ -25,7 +25,7 @@ void PyObjSnapshot::rehydrate() { PyObjRehydrator rehydrator; rehydrator.start(this); - rehydrator.finalize(); + rehydrator.rehydrateAll(); } std::string PyObjSnapshot::kindAsString() const { @@ -38,8 +38,8 @@ std::string PyObjSnapshot::kindAsString() const { if (mKind == Kind::NamedPyObject) { return "NamedPyObject"; } - if (mKind == Kind::Instance) { - return "Instance"; + if (mKind == Kind::PrimitiveInstance) { + return "PrimitiveInstance"; } if (mKind == Kind::ValueInstance) { return "ValueInstance"; @@ -285,6 +285,8 @@ void PyObjSnapshot::becomeInternalizedOf( } mNamedInts["type_is_forward"] = t->isForwardDefined() ? 1 : 0; + mName = t->name(); + mModuleName = t->moduleName(); if (t->isListOf()) { mKind = Kind::ListOfType; @@ -799,18 +801,18 @@ void PyObjSnapshot::becomeInternalizedOf( } if (val == Py_None) { - mKind = Kind::Instance; + mKind = Kind::PrimitiveInstance; return; } if (PyBool_Check(val)) { - mKind = Kind::Instance; + mKind = Kind::PrimitiveInstance; mInstance = Instance::create(val == Py_True); return; } if (PyLong_Check(val)) { - mKind = Kind::Instance; + mKind = Kind::PrimitiveInstance; try { mInstance = Instance::create((int64_t)PyLong_AsLongLong(val)); @@ -823,7 +825,7 @@ void PyObjSnapshot::becomeInternalizedOf( } if (PyFloat_Check(val)) { - mKind = Kind::Instance; + mKind = Kind::PrimitiveInstance; mInstance = Instance::create(PyFloat_AsDouble(val)); return; } @@ -851,7 +853,7 @@ void PyObjSnapshot::becomeInternalizedOf( } if (PyBytes_Check(val)) { - mKind = Kind::Instance; + mKind = Kind::PrimitiveInstance; mInstance = Instance::createAndInitialize( BytesType::Make(), [&](instance_ptr i) { @@ -1036,15 +1038,21 @@ void PyObjSnapshot::becomeInternalizedOf( return; } + if (val.type()->isString()) { + mKind = Kind::String; + mStringValue = ((StringType*)val.type())->toUtf8String(val.data()); + return; + } + if (val.type()->isNone() || val.type()->isBytes() || val.type()->isRegister()) { - mKind = Kind::Instance; + mKind = Kind::PrimitiveInstance; mInstance = val; return; } throw std::runtime_error( "Can't make a PyObjSnapshot out of an instance of type " - + mInstance.type()->name() + + val.type()->name() ); } @@ -1226,3 +1234,173 @@ PyObjSnapshot* PyObjSnapshot::computeForwardTargetTransitive() { } } + +void PyObjSnapshot::recomputeRecursiveName(const std::unordered_set& group) { + SnapshotStack stack; + + mName = computeRecursiveName(stack, group); +} + +std::string PyObjSnapshot::computeRecursiveName(SnapshotStack& stack, const std::unordered_set& group) { + if (mKind == Kind::PrimitiveType) { + return mType->name(); + } + + // if we're not in the group, then we will have a name computed already + if (group.find(this) == group.end()) { + return mName; + } + + // the names of these classes can't change - they are given to us + if (mKind == Kind::ClassType || mKind == Kind::HeldClassType || mKind == Kind::AlternativeType) { + return mName; + } + + long index = stack.indexOf(this); + if (index != -1) { + return "^" + format(index); + } + + PushSnapshotStack push(stack, this); + + if (mKind == Kind::PythonObjectOfTypeType) { + return "PythonObjectOfType(" + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + ")"; + } + + if (mKind == Kind::AlternativeMatcherType) { + return "AlternativeMatcher(" + getNamedElement("alternative", false)->computeRecursiveName(stack, group) + ")"; + } + + if (mKind == Kind::ConcreteAlternativeType) { + return getNamedElement("alternative", false)->computeRecursiveName(stack, group) + "." + + getNamedElement("alternative", false)->getNameByIx(mNamedInts["which"]); + } + + if (mKind == Kind::BoundMethodType) { + if (mNames.size() != 1) { + throw std::runtime_error("Corrupt PyObjSnapshot.BoundMethodType: no name"); + } + + return "BoundMethod(" + getNamedElement("self_type", false)->computeRecursiveName(stack, group) + ", " + mNames[0] + ")"; + } + + if (mKind == Kind::NamedTupleType) { + std::string name = "NamedTuple("; + + for (long k = 0; k < mElements.size() && k < mNames.size(); k++) { + if (k) { + name += ", "; + } + name += mNames[k] + "=" + mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::TupleType) { + std::string name = "Tuple("; + + for (long k = 0; k < mElements.size() && k < mNames.size(); k++) { + if (k) { + name += ", "; + } + name += mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::DictType) { + return "Dict(" + + getNamedElement("key_type", false)->computeRecursiveName(stack, group) + + ", " + + getNamedElement("value_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::ConstDictType) { + return "ConstDict(" + + getNamedElement("key_type", false)->computeRecursiveName(stack, group) + + ", " + + getNamedElement("value_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::ListOfType) { + return "ListOf(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::SetType) { + return "Set(" + + getNamedElement("key_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::TypedCellType) { + return "TypedCell(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::TupleOfType) { + return "TupleOf(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::PointerToType) { + return "PointerTo(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::SubclassOfTypeType) { + return "SubclassOf(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::RefToType) { + return "RefTo(" + + getNamedElement("element_type", false)->computeRecursiveName(stack, group) + + ")"; + } + + if (mKind == Kind::OneOfType) { + std::string name = "mElements("; + + for (long k = 0; k < mElements.size(); k++) { + if (k) { + name += ", "; + } + name += mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::OneOfType) { + std::string name = "mElements("; + + for (long k = 0; k < mElements.size(); k++) { + if (k) { + name += ", "; + } + name += mElements[k]->computeRecursiveName(stack, group); + } + + name += ")"; + return name; + } + + if (mKind == Kind::ForwardType) { + return mName; + } + + return ""; +} diff --git a/typed_python/PyObjSnapshot.hpp b/typed_python/PyObjSnapshot.hpp index 8eead9d1f..67d7d719c 100644 --- a/typed_python/PyObjSnapshot.hpp +++ b/typed_python/PyObjSnapshot.hpp @@ -21,6 +21,7 @@ #include "Instance.hpp" #include "PythonTypeInternals.hpp" #include "PyObjSnapshotMaker.hpp" +#include "TypeStack.hpp" class PyObjGraphSnapshot; class PyObjSnapshot; @@ -29,6 +30,9 @@ class FunctionOverload; class PyObjRehydrator; class PyObjSnapshotMaker; +typedef PtrStack SnapshotStack; +typedef PushPtrStack PushSnapshotStack; + /********************************* PyObjSnapshot @@ -361,13 +365,17 @@ class PyObjSnapshot { return mNamedElements.find(s) != mNamedElements.end(); } - PyObjSnapshot* getNamedElement(std::string s) const { + PyObjSnapshot* getNamedElement(std::string s, bool allowEmpty = true) const { auto it = mNamedElements.find(s); if (it != mNamedElements.end()) { return it->second; } + if (!allowEmpty) { + throw std::runtime_error("Can't find element named '" + s + "'."); + } + return nullptr; } @@ -502,7 +510,7 @@ class PyObjSnapshot { return mPyObject; } - if (mKind == Kind::Instance) { + if (mKind == Kind::PrimitiveInstance) { mPyObject = PyInstance::extractPythonObject(mInstance); return mPyObject; } @@ -517,13 +525,13 @@ class PyObjSnapshot { std::string inner; if (mType) { inner = mType->name(); - } else + } else if (mName.size()) { inner = mName; if (mModuleName.size()) { inner = mModuleName + "." + mName; } - } else + } else if (mPyObject) { inner = std::string("of type ") + mPyObject->ob_type->tp_name; } @@ -577,7 +585,7 @@ class PyObjSnapshot { if (mKind == Kind::PrimitiveType) { context.serializeNativeType(mType, buffer, 2); } else - if (mKind == Kind::Instance) { + if (mKind == Kind::PrimitiveInstance) { buffer.writeBeginCompound(3); context.serializeNativeType(mInstance.type(), buffer, 0); mInstance.type()->serialize(mInstance.data(), buffer, 1); @@ -662,7 +670,7 @@ class PyObjSnapshot { void clearCache() { if (mKind == Kind::ArbitraryPyObject - || mKind == Kind::Instance + || mKind == Kind::PrimitiveInstance || mKind == Kind::PrimitiveType ) { throw std::runtime_error("Can't clear the cache of a leaf node."); @@ -724,6 +732,14 @@ class PyObjSnapshot { return updated; } + std::string getNameByIx(long ix) { + if (ix < 0 || ix >= mNames.size()) { + throw std::runtime_error("NameIndex out of bounds."); + } + + return mNames[ix]; + } + PyObjSnapshot* computeForwardTarget() { if (mKind != Kind::ForwardType) { return nullptr; @@ -747,6 +763,10 @@ class PyObjSnapshot { PyObjSnapshot* computeForwardTargetTransitive(); + std::string computeRecursiveName(SnapshotStack& stack, const std::unordered_set& group); + + void recomputeRecursiveName(const std::unordered_set& group); + private: void markInternalizedOnType() { if (!mType) { diff --git a/typed_python/PyObjSnapshotGroupSearch.hpp b/typed_python/PyObjSnapshotGroupSearch.hpp index 965857b27..fcbeddbff 100644 --- a/typed_python/PyObjSnapshotGroupSearch.hpp +++ b/typed_python/PyObjSnapshotGroupSearch.hpp @@ -26,9 +26,13 @@ class PyObjSnapshotGroupSearch { { } - void add(PyObjSnapshot* o) { + void add(PyObjSnapshot* o, bool insistNew=false) { if (mSnapToOutGroupIx.find(o) != mSnapToOutGroupIx.end()) { - throw std::runtime_error("We've already seen this element"); + if (insistNew) { + throw std::runtime_error("We've already seen this element"); + } else { + return; + } } pushGroup(o); diff --git a/typed_python/PyPyObjSnapshot.cpp b/typed_python/PyPyObjSnapshot.cpp index 39204a5de..b11f93d18 100644 --- a/typed_python/PyPyObjSnapshot.cpp +++ b/typed_python/PyPyObjSnapshot.cpp @@ -157,10 +157,6 @@ PyObject* PyPyObjSnapshot::tp_getattro(PyObject* selfObj, PyObject* attrName) { return incref((PyObject*)PyInstance::typeObj(obj->getType())); } - if (attr == "instance" && obj->isInstance()) { - return PyInstance::fromInstance(obj->getInstance()); - } - if (attr == "stringValue" && obj->isString()) { return PyUnicode_FromString(obj->getStringValue().c_str()); } diff --git a/typed_python/PythonSerializationContext_deserialization.cpp b/typed_python/PythonSerializationContext_deserialization.cpp index 30043ee6a..de6db777f 100644 --- a/typed_python/PythonSerializationContext_deserialization.cpp +++ b/typed_python/PythonSerializationContext_deserialization.cpp @@ -645,7 +645,7 @@ Type* PythonSerializationContext::constructBlankSubclassOfTypeCategory(Type::Typ } if (result) { - result->markActivelyBeingDeserialized(isForwardDefined); + result->markActivelyBeingDeserialized(isForwardDefined, "Unknown", "Unknown"); return result; } diff --git a/typed_python/Type.cpp b/typed_python/Type.cpp index 337faa024..2a97abd03 100644 --- a/typed_python/Type.cpp +++ b/typed_python/Type.cpp @@ -441,6 +441,10 @@ void Type::attemptToResolve() { // type that it came from. graph.resolveForwards(); + // recompute the names of all the recursive types in here, which we have to do before + // rehydrating anything + graph.recomputeNames(); + // now take every non-forward node and make a version of it in the internal graph, // if it doesn't already exist. Then rehydrate it. At this point, we have an interned // copy of every single type and pyobject that was present in this graph. diff --git a/typed_python/Type.hpp b/typed_python/Type.hpp index e8db84dc9..34c5a1388 100644 --- a/typed_python/Type.hpp +++ b/typed_python/Type.hpp @@ -317,35 +317,20 @@ class Type { ); } - const std::string& name(bool stripQualname=false) const { - if (stripQualname) { - if (!m_stripped_name.size()) { - m_stripped_name = qualname_to_name(m_name); - } - - return m_stripped_name; - } + const std::string& name() const { return m_name; } std::string moduleName() { - return this->check([&](auto& subtype) { - return subtype.moduleNameConcrete(); - }); - } - - std::string moduleNameConcrete() { - return "builtins"; + return m_module_name; } std::string nameWithModule() { - return this->check([&](auto& subtype) { - return subtype.nameWithModuleConcrete(); - }); - } + if (m_module_name.size()) { + return m_module_name + "." + m_name; + } - std::string nameWithModuleConcrete() { - return name(); + return m_name; } const char* doc() { @@ -933,9 +918,11 @@ class Type { m_is_forward_defined = isForwardDefined; } - void markActivelyBeingDeserialized(bool isForwardDefined) { + void markActivelyBeingDeserialized(bool isForwardDefined, std::string inModuleName, std::string inName) { m_is_being_deserialized = true; m_is_forward_defined = isForwardDefined; + m_name = inName; + m_module_name = inModuleName; } bool isActivelyBeingDeserialized() { @@ -989,11 +976,9 @@ class Type { bool m_is_default_constructible; - std::string m_recursive_name; - std::string m_name; - mutable std::string m_stripped_name; + std::string m_module_name; PyTypeObject* mTypeRep; @@ -1102,7 +1087,6 @@ class Type { void recomputeName() { TypeStack stack; m_name = computeRecursiveName(stack); - m_stripped_name = ""; } void internalize(); diff --git a/typed_python/TypeStack.hpp b/typed_python/TypeStack.hpp index b2eb5403d..9ae61b81a 100644 --- a/typed_python/TypeStack.hpp +++ b/typed_python/TypeStack.hpp @@ -25,10 +25,10 @@ class Type; // is above us in the stack and if so, how many levels up, using 'indexOf' // and push new types to the stack using PushTypeStack. The stack can't have // duplicates. - -class TypeStack { +template +class PtrStack { public: - long indexOf(Type* t) { + long indexOf(T* t) { auto it = mTypeIndices.find(t); if (it == mTypeIndices.end()) { @@ -38,7 +38,7 @@ class TypeStack { return mTypeIndices.size() - it->second - 1; } - void push(Type* t) { + void push(T* t) { if (mTypeIndices.find(t) != mTypeIndices.end()) { throw std::runtime_error("Type is already in the type stack"); } @@ -47,26 +47,32 @@ class TypeStack { mTypeIndices[t] = s; } - void pop(Type* t) { + void pop(T* t) { mTypeIndices.erase(t); } private: - std::map mTypeIndices; + std::map mTypeIndices; }; +typedef PtrStack TypeStack; + -class PushTypeStack { +template +class PushPtrStack { public: - PushTypeStack(TypeStack& stack, Type* t) : mStack(stack), mT(t) { + PushPtrStack(PtrStack& stack, T* t) : mStack(stack), mT(t) { stack.push(mT); } - ~PushTypeStack() { + ~PushPtrStack() { mStack.pop(mT); } private: - TypeStack& mStack; - Type* mT; + PtrStack& mStack; + T* mT; }; + + +typedef PushPtrStack PushTypeStack; diff --git a/typed_python/_runtime.cpp b/typed_python/_runtime.cpp index 6a0d8cddd..6f7933668 100644 --- a/typed_python/_runtime.cpp +++ b/typed_python/_runtime.cpp @@ -478,7 +478,7 @@ extern "C" { } StringType::layout* np_typePtrToName(Type* t) { - return StringType::createFromString(t->name(true)); + return StringType::createFromString(t->name()); } StringType::layout* np_typePtrToQualName(Type* t) {