/*****************************************************************************

* Copyright (c) 2007 Piyush verma <piyush.verma@gmail.com> *

* Copyright 2007 Andreas Pakulat <apaku@gmx.de> *

* Copyright 2010-2013 Sven Brauch <svenbrauch@googlemail.com> *

* *

* This program is free software; you can redistribute it and/or *

* modify it under the terms of the GNU General Public License as *

* published by the Free Software Foundation; either version 2 of *

* the License, or (at your option) any later version. *

* *

* This program is distributed in the hope that it will be useful, *

* but WITHOUT ANY WARRANTY; without even the implied warranty of *

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *

* GNU General Public License for more details. *

* *

* You should have received a copy of the GNU General Public License *

* along with this program. If not, see <http://www.gnu.org/licenses/>. *

*****************************************************************************

*/

#include "declarationbuilder.h"

#include "duchain/declarations/decorator.h"

#include "duchain/declarations/functiondeclaration.h"

#include "duchain/declarations/classdeclaration.h"

#include "types/hintedtype.h"

#include "types/unsuretype.h"

#include "types/indexedcontainer.h"

#include "contextbuilder.h"

#include "expressionvisitor.h"

#include "pythoneditorintegrator.h"

#include "helpers.h"

#include "assistants/missingincludeassistant.h"

#include "correctionhelper.h"

#include <language/duchain/functiondeclaration.h>

#include <language/duchain/declaration.h>

#include <language/duchain/duchain.h>

#include <language/duchain/types/alltypes.h>

#include <language/duchain/classdeclaration.h>

#include <language/duchain/declaration.h>

#include <language/duchain/builders/abstracttypebuilder.h>

#include <language/duchain/aliasdeclaration.h>

#include <language/duchain/duchainutils.h>

#include <language/backgroundparser/backgroundparser.h>

#include <language/backgroundparser/parsejob.h>

#include <interfaces/ilanguagecontroller.h>

#include <QByteArray>

#include <QtGlobal>

#include <QDebug>

#include "duchaindebug.h"

#include <KUrl>

#include <functional>

using namespace KTextEditor;

using namespace KDevelop;

namespace Python

{

DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority)

: DeclarationBuilderBase()

, m_ownPriority(ownPriority)

{

setEditor(editor);

qCDebug(KDEV_PYTHON_DUCHAIN) << "Building Declarations";

}

DeclarationBuilder:: ~DeclarationBuilder()

{

if ( ! m_scheduledForDeletion.isEmpty() ) {

DUChainWriteLocker lock;

foreach ( DUChainBase* d, m_scheduledForDeletion ) {

delete d;

}

m_scheduledForDeletion.clear();

}

}

void DeclarationBuilder::setPrebuilding(bool prebuilding)

{

m_prebuilding = prebuilding;

}

ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext)

{

m_correctionHelper.reset(new CorrectionHelper(url, this));

// The declaration builder needs to run twice, so it can resolve uses of e.g. functions

// which are called before they are defined (which is easily possible, due to python's dynamic nature).

if ( ! m_prebuilding ) {

qCDebug(KDEV_PYTHON_DUCHAIN) << "building, but running pre-builder first";

DeclarationBuilder* prebuilder = new DeclarationBuilder(editor());

prebuilder->m_ownPriority = m_ownPriority;

prebuilder->m_currentlyParsedDocument = currentlyParsedDocument();

prebuilder->setPrebuilding(true);

prebuilder->m_futureModificationRevision = m_futureModificationRevision;

updateContext = prebuilder->build(url, node, updateContext);

qCDebug(KDEV_PYTHON_DUCHAIN) << "pre-builder finished";

delete prebuilder;

}

else {

qCDebug(KDEV_PYTHON_DUCHAIN) << "prebuilding";

}

return DeclarationBuilderBase::build(url, node, updateContext);

}

int DeclarationBuilder::jobPriority() const

{

return m_ownPriority;

}

void DeclarationBuilder::closeDeclaration()

{

if ( lastContext() ) {

DUChainReadLocker lock(DUChain::lock());

currentDeclaration()->setKind(Declaration::Type);

}

Q_ASSERT(currentDeclaration()->alwaysForceDirect());

eventuallyAssignInternalContext();

DeclarationBuilderBase::closeDeclaration();

}

template<typename T> T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, Ast* range, FitDeclarationType mustFitType)

{

QList<Declaration*> existingDeclarations = existingDeclarationsForNode(name);

Declaration* dec = 0;

reopenFittingDeclaration<T>(existingDeclarations, mustFitType, editorFindRange(range, range), &dec);

bool declarationOpened = (bool) dec;

if ( ! declarationOpened ) {

dec = openDeclaration<T>(name, range);

}

Q_ASSERT(dynamic_cast<T*>(dec));

return static_cast<T*>(dec);

}

template<typename T> T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type)

{

if ( node->astType == Ast::NameAstType ) {

NameAst* currentVariableDefinition = static_cast<NameAst*>(node);

// those contexts can invoke a variable declaration

// this prevents "bar" from being declared in something like "foo = bar"

// This is just a sanity check, the code should never request creation of a variable

// in such cases.

QList<ExpressionAst::Context> declaringContexts;

declaringContexts << ExpressionAst::Store << ExpressionAst::Parameter << ExpressionAst::AugStore;

if ( ! declaringContexts.contains(currentVariableDefinition->context) ) {

return 0;

}

Identifier* id = currentVariableDefinition->identifier;

return visitVariableDeclaration<T>(id, currentVariableDefinition, previous, type);

}

else if ( node->astType == Ast::IdentifierAstType ) {

return visitVariableDeclaration<T>(static_cast<Identifier*>(node), 0, previous, type);

}

else {

qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error";

return static_cast<T*>(0);

}

}

template<typename T> T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, RangeInRevision range,

AbstractType::Ptr type)

{

Ast pseudo;

pseudo.startLine = range.start.line; pseudo.startCol = range.start.column;

pseudo.endLine = range.end.line; pseudo.endCol = range.end.column;

T* result = visitVariableDeclaration<T>(node, &pseudo, 0, type);

return result;

}

QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node)

{

QList<Declaration*> existingDeclarations = currentContext()->findDeclarations(

identifierForNode(node).last(), CursorInRevision::invalid(), 0,

(DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases)

);

// append arguments context

if ( m_mostRecentArgumentsContext ) {

QList<Declaration*> args = m_mostRecentArgumentsContext->findDeclarations(

identifierForNode(node).last(), CursorInRevision::invalid(), 0, DUContext::DontSearchInParent

);

existingDeclarations.append(args);

}

return existingDeclarations;

}

DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias)

{

if ( type ) {

if ( type->whichType() == AbstractType::TypeFunction ) {

return FunctionDeclarationType;

}

}

if ( isAlias ) {

return AliasDeclarationType;

}

return InstanceDeclarationType;

}

template<typename T> QList<Declaration*> DeclarationBuilder::reopenFittingDeclaration(

QList<Declaration*> declarations, FitDeclarationType mustFitType,

RangeInRevision updateRangeTo, Declaration** ok )

{

// Search for a declaration from a previous parse pass which should be re-used

QList<Declaration*> remainingDeclarations;

*ok = 0;

foreach ( Declaration* d, declarations ) {

Declaration* fitting = dynamic_cast<T*>(d);

if ( ! fitting ) {

// Only use a declaration if the type matches

qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type";

continue;

}

// Do not use declarations which have been encountered previously;

// this function only handles declarations from previous parser passes which have not

// been encountered yet in this pass

bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d);

bool invalidType = false;

if ( d && d->abstractType() && mustFitType != NoTypeRequired ) {

invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) );

if ( ! invalidType ) {

invalidType = ( ( dynamic_cast<AliasDeclaration*>(d) != 0 ) != ( mustFitType == AliasDeclarationType ) );

}

}

if ( fitting && ! reallyEncountered && ! invalidType ) {

if ( d->topContext() == currentContext()->topContext() ) {

openDeclarationInternal(d);

d->setRange(updateRangeTo);

*ok = d;

setEncountered(d);

break;

}

else {

qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context";

}

}

else if ( ! invalidType ) {

remainingDeclarations << d;

}

}

return remainingDeclarations;

}

typedef QPair<Declaration*, int> p;

template<typename T> T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Ast* originalAst, Declaration* previous, AbstractType::Ptr type)

{

DUChainWriteLocker lock;

Ast* rangeNode = originalAst ? originalAst : node;

RangeInRevision range = editorFindRange(rangeNode, rangeNode);

// ask the correction file library if there's a user-specified type for this object

if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) {

type = hint;

}

// If no type is known, display "mixed".

if ( ! type ) {

type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed));

}

QList<Declaration*> existingDeclarations;

if ( previous ) {

existingDeclarations << previous;

}

else {

// declarations declared at an earlier range in this top-context

existingDeclarations = existingDeclarationsForNode(node);

}

// declaration existing in a previous version of this top-context

Declaration* dec = 0;

existingDeclarations = reopenFittingDeclaration<T>(existingDeclarations, kindForType(type), range, &dec);

bool declarationOpened = (bool) dec;

// tells whether the declaration found for updating is in the same top context

bool inSameTopContext = true;

// tells whether there's fitting declarations to update (update is not the same as re-open! one is for

// code which uses the same variable twice, the other is for multiple passes of the parser)

bool haveFittingDeclaration = false;

if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) {

Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last());

DUChainReadLocker lock;

if ( d && d->topContext() != topContext() ) {

inSameTopContext = false;

}

if ( dynamic_cast<T*>(existingDeclarations.last()) ) {

haveFittingDeclaration = true;

}

}

if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) {

// If the current context is a class, then this is a class member variable.

if ( ! dec ) {

dec = openDeclaration<ClassMemberDeclaration>(identifierForNode(node), range);

Q_ASSERT(! declarationOpened);

declarationOpened = true;

}

if ( declarationOpened ) {

DeclarationBuilderBase::closeDeclaration();

}

dec->setType(AbstractType::Ptr(type));

dec->setKind(KDevelop::Declaration::Instance);

} else if ( ! haveFittingDeclaration ) {

// This name did not previously appear in the user code, so a new variable is declared

// check whether a declaration from a previous parser pass must be updated

if ( ! dec ) {

dec = openDeclaration<T>(identifierForNode(node), range);

Q_ASSERT(! declarationOpened);

declarationOpened = true;

}

if ( declarationOpened ) {

DeclarationBuilderBase::closeDeclaration();

}

AbstractType::Ptr newType;

if ( currentContext()->type() == DUContext::Function ) {

// check for argument type hints (those are created when calling functions)

AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType(), topContext());

qCDebug(KDEV_PYTHON_DUCHAIN) << hints->toString();

if ( hints.cast<IndexedContainer>() || hints.cast<ListType>() ) {

// This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed.

newType = hints;

}

else {

newType = Helper::mergeTypes(hints, type);

}

}

else {

newType = type;

}

dec->setType(newType);

dec->setKind(KDevelop::Declaration::Instance);

}

else if ( inSameTopContext ) {

// The name appeared previously in the user code, so no new variable is declared, but just

// the type is modified accordingly.

dec = existingDeclarations.last();

AbstractType::Ptr currentType = dec->abstractType();

AbstractType::Ptr newType = type;

if ( newType ) {

if ( currentType && currentType->indexed() != newType->indexed() ) {

// If the previous and new type are different, use an unsure type

dec->setType(Helper::mergeTypes(currentType, newType));

}

else {

// If no type was set previously, use only the new one.

dec->setType(AbstractType::Ptr(type));

}

}

}

T* result = dynamic_cast<T*>(dec);

if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type";

return result;

}

void DeclarationBuilder::visitCode(CodeAst* node)

{

Q_ASSERT(currentlyParsedDocument().toUrl().isValid());

m_unresolvedImports.clear();

DeclarationBuilderBase::visitCode(node);

}

void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node)

{

if ( node->name ) {

// Python allows to assign the caught exception to a variable; create that variable if required.

ExpressionVisitor v(currentContext());

v.visitNode(node->type);

visitVariableDeclaration<Declaration>(node->name, 0, v.lastType());

}

DeclarationBuilderBase::visitExceptionHandler(node);

}

void DeclarationBuilder::visitWithItem(WithItemAst* node)

{

if ( node->optionalVars ) {

// For statements like "with open(f) as x", a new variable must be created; do this here.

ExpressionVisitor v(currentContext());

v.visitNode(node->contextExpression);

visitVariableDeclaration<Declaration>(node->optionalVars, 0, v.lastType());

}

Python::AstDefaultVisitor::visitWithItem(node);

}

void DeclarationBuilder::visitFor(ForAst* node)

{

ExpressionVisitor v(currentContext());

v.visitNode(node->iterator);

auto possibleIterators = Helper::filterType<ListType>(v.lastType(),

[](AbstractType::Ptr type) {

auto container = type.cast<ListType>();

return container && container->contentType();

}

);

if ( node->target->astType == Ast::NameAstType ) {

// In case the iterator variable is a Name ("for x in range(3)"), just create a declaration for it.

// The following code tries to figure out the type of "x" from the object that is being iterated over.

auto iteratorType = Helper::foldTypes<ListType::Ptr>(possibleIterators,

[](const ListType::Ptr& p) {

return p->contentType().abstractType();

}

);

// otherwise, no list type whatsoever was available for the iterator list, so just display "mixed".

// Create the variable declaration for the iterator variable with the type that has been determined.

visitVariableDeclaration<Declaration>(node->target, 0, iteratorType);

}

else if ( node->target->astType == Ast::TupleAstType ) {

// If the target is a tuple ("for x, y, z in ..."), multiple variables must be declared.

// For now, types of those variables will only be determined if the iterator is a list of tuples.

QList<ExpressionAst*> targetElements = targetsOfAssignment(QList<ExpressionAst*>() << node->target);

int targetElementsCount = targetElements.count();

QList<IndexedContainer::Ptr> gatherFromTuples;

for ( auto container : possibleIterators ) {

AbstractType::Ptr contentType = container->contentType().abstractType();

gatherFromTuples = Helper::filterType<IndexedContainer>(contentType,

// find all IndexedContainer entries which have the right number of entries

[targetElementsCount](AbstractType::Ptr type) {

IndexedContainer::Ptr indexed = type.cast<IndexedContainer>();

return indexed && indexed->typesCount() == targetElementsCount;

}

);

}

// Now, iterate over all possible tuples, and extract their types

int i = 0;

QList<AbstractType::Ptr> targetTypes;

foreach ( IndexedContainer::Ptr tuple, gatherFromTuples ) {

for ( int j = 0; j < tuple->typesCount(); j++ ) {

if ( i == 0 ) {

targetTypes.append(tuple->typeAt(j).abstractType());

}

else {

targetTypes[j] = Helper::mergeTypes(targetTypes[j], tuple->typeAt(j).abstractType());

}

}

i++;

}

short atElement = 0;

bool haveTypeInformation = ! targetTypes.isEmpty();

Q_ASSERT( ! haveTypeInformation || targetTypes.length() == targetElementsCount );

foreach ( ExpressionAst* tupleMember, targetElements ) {

if ( tupleMember->astType == Ast::NameAstType ) {

AbstractType::Ptr newType;

if ( haveTypeInformation ) {

newType = targetTypes.at(atElement);

}

else {

newType = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed));

}

visitVariableDeclaration<Declaration>(tupleMember, 0, newType);

}

++atElement;

}

}

Python::ContextBuilder::visitFor(node);

}

Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const

{

DUChainReadLocker lock(DUChain::lock());

DUContext* currentContext = ctx;

// TODO make this a bit faster, it wastes time

Declaration* lastAccessedDeclaration = 0;

int i = 0;

int identifierCount = dottedNameIdentifier.length();

foreach ( const QString& currentIdentifier, dottedNameIdentifier ) {

Q_ASSERT(currentContext);

i++;

QList<Declaration*> declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(),

CursorInRevision::invalid(), 0, DUContext::NoFiltering);

// break if the list of identifiers is not yet totally worked through and no

// declaration with an internal context was found

if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) {

qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path();

return 0;

}

else {

lastAccessedDeclaration = declarations.last();

currentContext = lastAccessedDeclaration->internalContext();

}

}

return lastAccessedDeclaration;

}

QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const

{

QString moduleName = alias->name->value;

if ( ! intermediate.isEmpty() ) {

moduleName.prepend('.').prepend(intermediate);

}

if ( node->module ) {

moduleName.prepend('.').prepend(node->module->value);

}

// To handle relative imports correctly, add node level in the beginning of the path

// This will allow findModulePath to deduce module search direcotry properly

moduleName.prepend(QString(node->level, '.'));

return moduleName;

}

void DeclarationBuilder::visitImportFrom(ImportFromAst* node)

{

Python::AstDefaultVisitor::visitImportFrom(node);

QString moduleName;

QString declarationName;

foreach ( AliasAst* name, node->names ) {

// iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf"

Identifier* declarationIdentifier = 0;

declarationName.clear();

if ( name->asName ) {

// use either the alias ("as foo"), or the object name itself if no "as" is given

declarationIdentifier = name->asName;

declarationName = name->asName->value;

}

else {

declarationIdentifier = name->name;

declarationName = name->name->value;

}

// This is a bit hackish, it tries to find the specified object twice twice -- once it tries to

// import the name from a module's __init__.py file, and once from a "real" python file

// TODO improve this code-wise

ProblemPointer problem(0);

QString intermediate;

moduleName = buildModuleNameFromNode(node, name, intermediate);

Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem);

if ( ! success && (node->module || node->level) ) {

ProblemPointer problem_init(0);

intermediate = QString("__init__");

moduleName = buildModuleNameFromNode(node, name, intermediate);

success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init);

}

if ( ! success && problem ) {

DUChainWriteLocker lock;

topContext()->addProblem(problem);

}

}

}

void DeclarationBuilder::visitComprehension(ComprehensionAst* node)

{

Python::AstDefaultVisitor::visitComprehension(node);

// make the declaration zero chars long; it must appear at the beginning of the context,

// because it is actually used *before* its real declaration: [foo for foo in bar]

// The DUChain doesn't like this, so for now, the declaration is at the opening bracket,

// and both other occurences are uses of that declaration.

// TODO add a special case to the usebuilder to display the second occurence as a declaration

RangeInRevision declarationRange(currentContext()->range().start, currentContext()->range().start);

declarationRange.end.column -= 1;

AbstractType::Ptr targetType(new IntegralType(IntegralType::TypeMixed));

if ( node->iterator ) {

// try to find the type of the object being iterated over, for guessing the

// type of the iterator variable

ExpressionVisitor v(currentContext());

v.visitNode(node->iterator);

if ( auto container = ListType::Ptr::dynamicCast(v.lastType()) ) {

targetType = container->contentType().abstractType();

}

}

// create variable declarations for the iterator variable(s)

if ( node->target->astType == Ast::NameAstType ) {

visitVariableDeclaration<Declaration>(static_cast<NameAst*>(node->target)->identifier, declarationRange, targetType);

}

if ( node->target->astType == Ast::TupleAstType ) {

foreach ( ExpressionAst* tupleElt, static_cast<TupleAst*>(node->target)->elements ) {

if ( tupleElt->astType == Ast::NameAstType ) {

NameAst* n = static_cast<NameAst*>(tupleElt);

visitVariableDeclaration<Declaration>(n->identifier, declarationRange);

}

}

}

}

void DeclarationBuilder::visitImport(ImportAst* node)

{

Python::ContextBuilder::visitImport(node);

DUChainWriteLocker lock;

foreach ( AliasAst* name, node->names ) {

QString moduleName = name->name->value;

// use alias if available, name otherwise

Identifier* declarationIdentifier = name->asName ? name->asName : name->name;

ProblemPointer problem(0);

createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem);

if ( problem ) {

DUChainWriteLocker lock;

topContext()->addProblem(problem);

}

}

}

void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule)

{

if ( doschedule ) {

m_scheduledForDeletion.append(d);

}

else {

m_scheduledForDeletion.removeAll(d);

}

}

Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier,

const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration,

const RangeInRevision& range)

{

// This actually handles two use cases which are very similar -- thus this check:

// There might be either one declaration which should be imported from another module,

// or there might be a whole context. In "import foo.bar", the "bar" might be either

// a single class/function/whatever, or a whole file to import.

// NOTE: The former case can't actually happen in python, it's not allowed. However,

// it is still handled here, because it's very useful for documentation files (pyQt for example

// makes heavy use of that feature).

Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided");

Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided");

qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents;

Declaration* lastDeclaration = 0;

int depth = 0;

// check for already existing trees to update

for ( int i = nameComponents.length() - 1; i >= 0; i-- ) {

QStringList currentName;

for ( int j = 0; j < i; j++ ) {

currentName.append(nameComponents.at(j));

}

lastDeclaration = findDeclarationInContext(currentName, topContext());

if ( lastDeclaration && lastDeclaration->range() < range ) {

depth = i;

break;

}

}

DUContext* extendingPreviousImportCtx = 0;

QStringList remainingNameComponents;

bool injectingContext = false;

if ( lastDeclaration && lastDeclaration->internalContext() ) {

qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value;

for ( int i = depth; i < nameComponents.length(); i++ ) {

remainingNameComponents.append(nameComponents.at(i));

}

extendingPreviousImportCtx = lastDeclaration->internalContext();

injectContext(extendingPreviousImportCtx);

injectingContext = true;

qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents;

}

else {

remainingNameComponents = nameComponents;

extendingPreviousImportCtx = topContext();

}

// now, proceed in creating the declaration tree with whatever context

QList<Declaration*> openedDeclarations;

QList<StructureType::Ptr> openedTypes;

QList<DUContext*> openedContexts;

RangeInRevision displayRange = RangeInRevision::invalid();

DUChainWriteLocker lock;

for ( int i = 0; i < remainingNameComponents.length(); i++ ) {

// Iterate over all the names, and create a declaration + sub-context for each of them

const QString& component = remainingNameComponents.at(i);

Identifier temporaryIdentifier(component);

Declaration* d = 0;

temporaryIdentifier.copyRange(declarationIdentifier);

temporaryIdentifier.endCol = temporaryIdentifier.startCol;

temporaryIdentifier.startCol += 1;

displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme

bool done = false;

if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) {

// it's the last level, so if we have an alias declaration create it and stop

if ( aliasDeclaration->isFunctionDeclaration()

|| dynamic_cast<ClassDeclaration*>(aliasDeclaration)

|| dynamic_cast<AliasDeclaration*>(aliasDeclaration)

) {

aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration);

AliasDeclaration* adecl = eventuallyReopenDeclaration<AliasDeclaration>(&temporaryIdentifier,

&temporaryIdentifier,

AliasDeclarationType);

if ( adecl ) {

adecl->setAliasedDeclaration(aliasDeclaration);

}

d = adecl;

closeDeclaration();

}

else {

d = visitVariableDeclaration<Declaration>(&temporaryIdentifier);

d->setAbstractType(aliasDeclaration->abstractType());

}

openedDeclarations.append(d);

done = true;

}

if ( ! done ) {

// create the next level of the tree hierarchy if not done yet.

d = visitVariableDeclaration<Declaration>(&temporaryIdentifier);

}

if ( d ) {

if ( topContext() != currentContext() ) {

d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start));

}

else {

d->setRange(displayRange);

}

d->setAutoDeclaration(true);

currentContext()->createUse(d->ownIndex(), displayRange);

qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion;

qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString();

scheduleForDeletion(d, false);

qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion;

}

if ( done ) break;

qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component;

// otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items)

StructureType::Ptr moduleType = StructureType::Ptr(new StructureType());

openType(moduleType);

openedContexts.append(openContext(declarationIdentifier, KDevelop::DUContext::Other));

foreach ( Declaration* local, currentContext()->localDeclarations() ) {

// keep all the declarations until the builder finished

// kdevelop would otherwise delete them as soon as the context is closed

if ( ! wasEncountered(local) ) {

setEncountered(local);

scheduleForDeletion(local, true);

}

}

openedDeclarations.append(d);

openedTypes.append(moduleType);

if ( i == remainingNameComponents.length() - 1 ) {

if ( innerCtx ) {

qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration";

currentContext()->addImportedParentContext(innerCtx);

}

else if ( aliasDeclaration ) {

qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration";

}

}

}

for ( int i = openedContexts.length() - 1; i >= 0; i-- ) {

// Close all the declarations and contexts opened previosly, and assign the types.

qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context";

closeType();

closeContext();

Declaration* d = openedDeclarations.at(i);

// because no context will be opened for an alias declaration, this will not happen if there's one

if ( d ) {

openedTypes[i]->setDeclaration(d);

d->setType(openedTypes.at(i));

d->setInternalContext(openedContexts.at(i));

}

}

if ( injectingContext ) {

closeInjectedContext();

}

if ( ! openedDeclarations.isEmpty() ) {

// return the lowest-level element in the tree, for the caller to do stuff with

return openedDeclarations.last();

}

else return 0;

}

Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName,

Identifier* declarationIdentifier,

ProblemPointer& problemEncountered, Ast* rangeNode)

{

// Search the disk for a python file which contains the requested declaration

QPair<KUrl, QStringList> moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl());

RangeInRevision range(RangeInRevision::invalid());

if ( rangeNode ) {

range = rangeForNode(rangeNode, false);

}

else {

range = rangeForNode(declarationIdentifier, false);

}

Q_ASSERT(range.isValid());

qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo;

qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value;

DUChainWriteLocker lock;

const IndexedString modulePath = IndexedString(moduleInfo.first);

ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath);

lock.unlock();

Declaration* resultingDeclaration = 0;

if ( ! moduleInfo.first.isValid() ) {

// The file was not found -- this is either an error in the user's code,

// a missing module, or a C module (.so) which is unreadable for kdevelop

// TODO imrpove error handling in case the module exists as a shared object or .pyc file only

qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo;

KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument());

p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange()));

p->setSource(KDevelop::ProblemData::SemanticAnalysis);

p->setSeverity(KDevelop::ProblemData::Warning);

p->setDescription(i18n("Module \"%1\" not found", moduleName));

problemEncountered = p;

return 0;

}

if ( ! moduleContext ) {

// schedule the include file for parsing, and schedule the current one for reparsing after that is done

qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling";

m_unresolvedImports.append(modulePath);

Helper::scheduleDependency(modulePath, m_ownPriority);

// parseDocuments() must *not* be called from a background thread!

// KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments();

return 0;

}

if ( moduleInfo.second.isEmpty() ) {

// import the whole module

resultingDeclaration = createDeclarationTree(declarationName.split("."),

declarationIdentifier, moduleContext, 0, range);

}

else {

// import a specific declaration from the given file

lock.lock();

if ( declarationIdentifier->value == "*" ) {

qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module";

currentContext()->addImportedParentContext(moduleContext);

}

else {

qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second;

Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext);

if ( originalDeclaration ) {

DUChainWriteLocker lock(DUChain::lock());

resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier,

ReferencedTopDUContext(0), originalDeclaration,

editorFindRange(declarationIdentifier, declarationIdentifier));

}

else {

KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument());

p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok?

p->setSource(KDevelop::ProblemData::SemanticAnalysis);

p->setSeverity(KDevelop::ProblemData::Warning);

p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join(".")));

problemEncountered = p;

}

}

}

return resultingDeclaration;

}

void DeclarationBuilder::visitYield(YieldAst* node)

{

// Functions containing "yield" statements will return lists in our abstraction.

// The content type of that list can be guessed from the yield statements.

AstDefaultVisitor::visitYield(node);

// Determine the type of the argument to "yield", like "int" in "yield 3"

ExpressionVisitor v(currentContext());

v.visitNode(node->value);

AbstractType::Ptr encountered = v.lastType();

// In some obscure (or wrong) cases, "yield" might appear outside of a function body,

// so check for that here.

if ( ! node->value || ! hasCurrentType() ) {

return;

}

TypePtr<FunctionType> t = currentType<FunctionType>();

if ( ! t ) {

return;

}

if ( auto previous = t->returnType().cast<ListType>() ) {

// If the return type of the function already is set to a list, *add* the encountered type

// to its possible content types.

previous->addContentType<Python::UnsureType>(encountered);

t->setReturnType(previous.cast<AbstractType>());

}

else {

// Otherwise, create a new container type, and set it as the function's return type.

DUChainWriteLocker lock;

auto container = ExpressionVisitor::typeObjectForIntegralType<ListType>("list", currentContext());

if ( container ) {

openType<ListType>(container);

container->addContentType<Python::UnsureType>(encountered);

t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast<AbstractType>()));

closeType();

}

}

}

void DeclarationBuilder::visitLambda(LambdaAst* node)

{

Python::AstDefaultVisitor::visitLambda(node);

DUChainWriteLocker lock;

// A context must be opened, because the lamdba's arguments are local to the lambda:

// d = lambda x: x*2; print x # <- gives an error

openContext(node, editorFindRange(node, node->body), DUContext::Other);

foreach ( ArgAst* argument, node->arguments->arguments ) {

visitVariableDeclaration<Declaration>(argument->argumentName);

}

closeContext();

}

void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function)

{

ExpressionVisitor v(currentContext());

v.visitNode(static_cast<AttributeAst*>(node->function)->value);

// Don't do anything if the object the function is being called on is not a container.

auto container = v.lastType().cast<ListType>();

if ( ! container || ! function ) {

return;

}

// Don't to updates to pre-defined functions.

if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) {

return;

}

// Check for the different types of modifiers such a function can have

QStringList args;

QHash< QString, std::function<void()> > items;

items["addsTypeOfArg"] = [&]() {

const int offset = ! args.isEmpty() ? args.at(0).toInt() : 0;

if ( node->arguments.length() <= offset ) {

return;

}

// Check which type should be added to the list

ExpressionVisitor argVisitor(currentContext());

argVisitor.visitNode(node->arguments.at(offset));

// Actually add that type

if ( ! argVisitor.lastType() ) {

return;

}

DUChainWriteLocker wlock;

qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString();

container->addContentType<Python::UnsureType>(argVisitor.lastType());

v.lastDeclaration()->setType(container);

};

items["addsTypeOfArgContent"] = [&]() {

const int offset = ! args.isEmpty() ? args.at(0).toInt() : 0;

if ( node->arguments.length() <= offset ) {

return;

}

ExpressionVisitor argVisitor(currentContext());

argVisitor.visitNode(node->arguments.at(offset));

DUChainWriteLocker wlock;

if ( ! argVisitor.lastType() ) {

return;

}

auto sources = Helper::filterType<ListType>(

argVisitor.lastType(), [](AbstractType::Ptr type) {

return type.cast<ListType>();

}

);

for ( auto sourceContainer : sources ) {

if ( ! sourceContainer->contentType() ) {

continue;

}

container->addContentType<Python::UnsureType>(sourceContainer->contentType().abstractType());

v.lastDeclaration()->setType(container);

}

};

foreach ( const QString& key, items.keys() ) {

if ( Helper::docstringContainsHint(function.data(), key, &args) ) {

items[key]();

}

}

}

void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer function)

{

DUChainReadLocker lock;

QPair<FunctionDeclaration::Ptr, bool> called = Helper::functionDeclarationForCalledDeclaration(function);

FunctionDeclaration::Ptr lastFunctionDeclaration = called.first;

bool isConstructor = called.second;

if ( ! lastFunctionDeclaration ) {

return;

}

if ( lastFunctionDeclaration->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) {

return;

}

DUContext* args = DUChainUtils::getArgumentContext(lastFunctionDeclaration.data());

FunctionType::Ptr functiontype = lastFunctionDeclaration->type<FunctionType>();

if ( ! args || ! functiontype ) {

return;

}

// The declaration which was found is a function declaration, and has a valid arguments list assigned.

QVector<Declaration*> parameters = args->localDeclarations();

const int specialParamsCount = (lastFunctionDeclaration->vararg() > 0) + (lastFunctionDeclaration->kwarg() > 0);

// Look for the "self" in the argument list, the type of that should not be updated.

bool hasSelfArgument = false;

if ( ( lastFunctionDeclaration->context()->type() == DUContext::Class || isConstructor )