diff --git a/.gitlab/build-codecompass.sh b/.gitlab/build-codecompass.sh index 5ae65df5c..1796db840 100644 --- a/.gitlab/build-codecompass.sh +++ b/.gitlab/build-codecompass.sh @@ -41,6 +41,7 @@ export LDFLAGS="-Wl,-rpath-link,$DEPS_INSTALL_RUNTIME_DIR/openssl-install/lib "\ "-Wl,-rpath-link,$DEPS_INSTALL_RUNTIME_DIR/postgresql-install/lib" export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/odb-install/lib\ +:$DEPS_INSTALL_RUNTIME_DIR/python-install/lib\ :$LD_LIBRARY_PATH export CMAKE_PREFIX_PATH=$DEPS_INSTALL_RUNTIME_DIR/libgit2-install\ @@ -48,6 +49,7 @@ export CMAKE_PREFIX_PATH=$DEPS_INSTALL_RUNTIME_DIR/libgit2-install\ :$DEPS_INSTALL_RUNTIME_DIR/thrift-install\ :$DEPS_INSTALL_RUNTIME_DIR/boost-install\ :$DEPS_INSTALL_RUNTIME_DIR/llvm-install\ +:$DEPS_INSTALL_RUNTIME_DIR/python-install\ :$DEPS_INSTALL_RUNTIME_DIR/openldap-install export GTEST_ROOT=$DEPS_INSTALL_BUILD_DIR/gtest-install diff --git a/.gitlab/build-deps.sh b/.gitlab/build-deps.sh index c6beb0777..919173a9a 100644 --- a/.gitlab/build-deps.sh +++ b/.gitlab/build-deps.sh @@ -240,11 +240,14 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/openssl-install/lib:$LD_LIBRARY --quiet \ --prefix=$DEPS_INSTALL_RUNTIME_DIR/python-install \ --with-openssl=$DEPS_INSTALL_RUNTIME_DIR/openssl-install \ + --enable-shared \ --enable-optimizations make install --quiet --jobs $(nproc) rm -f $PACKAGES_DIR/Python-3.9.0.tar.xz export PATH=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin:$PATH +export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/python-install/lib:$LD_LIBRARY_PATH +# --enabled-shared: build libpython.so instead of libpython.a, must be on LD_LIBRARY_PATH ############## # LLVM/Clang # @@ -313,7 +316,7 @@ if [ ! -f $DEPS_INSTALL_RUNTIME_DIR/boost-install/lib/libboost_program_options.s ./bootstrap.sh \ --prefix=$DEPS_INSTALL_RUNTIME_DIR/boost-install \ - --with-python=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin/python + --with-python=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin/python3 ./b2 -j $(nproc) install rm -f $PACKAGES_DIR/boost_1_74_0.tar.gz diff --git a/.gitlab/cc-env.sh b/.gitlab/cc-env.sh index 47a1fbaa7..4567d3cde 100644 --- a/.gitlab/cc-env.sh +++ b/.gitlab/cc-env.sh @@ -8,6 +8,7 @@ DEPS_INSTALL_DIR=$ROOT_DIR/deps-runtime CC_INSTALL_DIR=$ROOT_DIR/cc-install export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\ +:$DEPS_INSTALL_DIR/libgit2-install/lib\ :$DEPS_INSTALL_DIR/openssl-install/lib\ :$DEPS_INSTALL_DIR/graphviz-install/lib\ :$DEPS_INSTALL_DIR/libmagic-install/lib\ @@ -17,8 +18,10 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\ :$DEPS_INSTALL_DIR/llvm-install/lib\ :$DEPS_INSTALL_DIR/libtool-install/lib\ :$DEPS_INSTALL_DIR/postgresql-install/lib\ +:$DEPS_INSTALL_DIR/python-install/lib\ :$DEPS_INSTALL_DIR/openldap-install/lib\ :$LD_LIBRARY_PATH +# Note: libgit2-install/lib required on Ubuntu; libgit2-install/lib64 on SUSE export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\ :$DEPS_INSTALL_DIR/ctags-install/bin\ @@ -30,6 +33,7 @@ export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\ export MAGIC=$DEPS_INSTALL_DIR/libmagic-install/share/misc/magic.mgc export JAVA_HOME=$DEPS_INSTALL_DIR/jdk-install +export PYTHONHOME=$DEPS_INSTALL_DIR/python-install if [ -f $ROOT_DIR/ccdb-tool/venv/bin/activate ]; then source $ROOT_DIR/ccdb-tool/venv/bin/activate diff --git a/.gitlab/ci.yml b/.gitlab/ci.yml index ef5f30ee3..e5aff3dfb 100644 --- a/.gitlab/ci.yml +++ b/.gitlab/ci.yml @@ -62,7 +62,7 @@ tarball suse-15: extends: .tarball image: opensuse/leap:15 cache: - key: "leap" + key: "leap-$CI_COMMIT_REF_SLUG" variables: GCC_VERSION: 9.3.0 ODB_VERSION: 2.5.0 @@ -85,7 +85,7 @@ tarball suse-42.1: extends: .tarball image: opensuse/archive:42.1 cache: - key: "malachite" + key: "malachite-$CI_COMMIT_REF_SLUG" variables: GCC_VERSION: 5.5.0 ODB_VERSION: 2.4.0 @@ -111,7 +111,7 @@ tarball ubuntu-16.04: extends: .tarball image: ubuntu:16.04 cache: - key: "xenial" + key: "xenial-$CI_COMMIT_REF_SLUG" variables: GCC_VERSION: 5.5.0 ODB_VERSION: 2.4.0 diff --git a/doc/deps.md b/doc/deps.md index 1fcddb86a..54f95cfe0 100644 --- a/doc/deps.md +++ b/doc/deps.md @@ -40,6 +40,7 @@ be installed from the official repository of the given Linux distribution. - **`libgtest-dev`**: For testing CodeCompass. ***See [Known issues](#known-issues)!*** - **`libldap2-dev`**: For LDAP authentication. +- **`python3.8`**: For python parsing. ## Quick guide @@ -52,7 +53,7 @@ known issues. sudo apt install git cmake make g++ gcc-7-plugin-dev libboost-all-dev \ llvm-10-dev clang-10 libclang-10-dev \ default-jdk libssl1.0-dev libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen \ - libldap2-dev libgtest-dev npm + libldap2-dev libgtest-dev npm python3.8 ``` #### Ubuntu 20.04 ("Focal Fossa") LTS diff --git a/model/include/model/buildaction.h b/model/include/model/buildaction.h index 382ca817a..8b086ae4c 100644 --- a/model/include/model/buildaction.h +++ b/model/include/model/buildaction.h @@ -25,6 +25,7 @@ struct BuildAction { Compile, Link, + Interpret, Other }; diff --git a/plugins/cpp/webgui/js/cppInfoTree.js b/plugins/cpp/webgui/js/cppInfoTree.js index e112f40b4..b2d32a1a6 100644 --- a/plugins/cpp/webgui/js/cppInfoTree.js +++ b/plugins/cpp/webgui/js/cppInfoTree.js @@ -291,6 +291,7 @@ function (model, viewHandler, util) { } var cppInfoTree = { + id: 'cpp-info-tree', render : function (elementInfo) { var ret = []; diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt new file mode 100644 index 000000000..166d56255 --- /dev/null +++ b/plugins/python/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(parser) +add_subdirectory(model) +add_subdirectory(service) + +install_webplugin(webgui) \ No newline at end of file diff --git a/plugins/python/model/CMakeLists.txt b/plugins/python/model/CMakeLists.txt new file mode 100644 index 000000000..bb4200716 --- /dev/null +++ b/plugins/python/model/CMakeLists.txt @@ -0,0 +1,17 @@ +set(ODB_SOURCES + include/model/pythonastnode.h + include/model/pythonclass.h + include/model/pythondocumentation.h + include/model/pythonentity.h + include/model/pythonfunction.h + include/model/pythonimport.h + include/model/pythoninheritance.h + include/model/pythontype.h + include/model/pythonvariable.h) + +generate_odb_files("${ODB_SOURCES}") + +add_odb_library(pythonmodel ${ODB_CXX_SOURCES}) +target_link_libraries(pythonmodel model) + +install_sql() diff --git a/plugins/python/model/include/model/pythonastnode.h b/plugins/python/model/include/model/pythonastnode.h new file mode 100644 index 000000000..0c91bd441 --- /dev/null +++ b/plugins/python/model/include/model/pythonastnode.h @@ -0,0 +1,164 @@ +#ifndef CC_MODEL_PYTHONASTNODE_H +#define CC_MODEL_PYTHONASTNODE_H + +#include + +#include +#include +#include + +#include + +#include + +namespace cc +{ +namespace model +{ + +typedef std::uint64_t PythonAstNodeId; + +#pragma db object +struct PythonAstNode +{ + enum class SymbolType + { + Variable, + Function, + Class, + Module, + Other + }; + + enum class AstType + { + Declaration, + Usage, + Other + }; + + virtual ~PythonAstNode() {} + + #pragma db id + PythonAstNodeId id = 0; + + std::string astValue; + + std::string qualifiedName; + + #pragma db null + FileLoc location; + + SymbolType symbolType = SymbolType::Other; + + AstType astType = AstType::Other; + + std::string toString() const; + + bool operator< (const PythonAstNode& other) const { return id < other.id; } + bool operator==(const PythonAstNode& other) const { return id == other.id; } +}; + +typedef std::shared_ptr PythonAstNodePtr; + +inline std::string symbolTypeToString(PythonAstNode::SymbolType type_) +{ + switch (type_) + { + case PythonAstNode::SymbolType::Variable: return "Variable"; + case PythonAstNode::SymbolType::Function: return "Function"; + case PythonAstNode::SymbolType::Class: return "Class"; + case PythonAstNode::SymbolType::Module: return "Module"; + case PythonAstNode::SymbolType::Other: return "Other"; + } + + return std::string(); +} + +inline std::string astTypeToString(PythonAstNode::AstType type) +{ + switch (type) { + case PythonAstNode::AstType::Usage: return "Usage"; + case PythonAstNode::AstType::Declaration: return "Declaration"; + case PythonAstNode::AstType::Other: return "Other"; + } + + return std::string(); +} + +inline std::string PythonAstNode::toString() const +{ + return std::string("PythonAstNode") + .append("\nid = ").append(std::to_string(id)) + .append("\nastValue = ").append(astValue) + .append("\nqualifiedName = ").append(qualifiedName) + .append("\nlocation = ").append(location.file->path).append(" (") + .append(std::to_string( + static_cast(location.range.start.line))).append(":") + .append(std::to_string( + static_cast(location.range.start.column))).append(" - ") + .append(std::to_string( + static_cast(location.range.end.line))).append(":") + .append(std::to_string( + static_cast(location.range.end.column))).append(")") + .append("\nsymbolType = ").append(symbolTypeToString(symbolType)) + .append("\nastType = ").append(astTypeToString(astType)); +} + +inline std::uint64_t createIdentifier(const PythonAstNode& astNode_) +{ + using SymbolTypeInt + = std::underlying_type::type; + using AstTypeInt + = std::underlying_type::type; + + std::string res; + + res + .append(astNode_.astValue).append(":") + .append(astNode_.qualifiedName).append(":") + .append(std::to_string(static_cast(astNode_.symbolType))).append(":") + .append(std::to_string(static_cast(astNode_.astType))).append(":"); + + if (astNode_.location.file){ + res + .append(std::to_string(astNode_.location.file->id)).append(":") + .append(std::to_string(astNode_.location.range.start.line)).append(":") + .append(std::to_string(astNode_.location.range.start.column)).append(":") + .append(std::to_string(astNode_.location.range.end.line)).append(":") + .append(std::to_string(astNode_.location.range.end.column)).append(":"); + } else { + res.append("null"); + } + + return util::fnvHash(res); +} + +inline std::uint64_t createAstNodeInfoEntityHash(const PythonAstNode& astNode_) +{ + return util::fnvHash(astNode_.qualifiedName); +} + +#pragma db view \ + object(PythonAstNode) object(File = LocFile : PythonAstNode::location.file) \ + query ((?) + "GROUP BY" + LocFile::id + "ORDER BY" + LocFile::id) +struct PythonAstCountGroupByFiles +{ + #pragma db column(LocFile::id) + FileId file; + + #pragma db column("count(" + PythonAstNode::id + ")") + std::size_t count; +}; + +#pragma db view object(PythonAstNode) +struct PythonAstCount +{ + #pragma db column("count(" + PythonAstNode::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonclass.h b/plugins/python/model/include/model/pythonclass.h new file mode 100644 index 000000000..3e5795446 --- /dev/null +++ b/plugins/python/model/include/model/pythonclass.h @@ -0,0 +1,77 @@ +#ifndef CC_MODEL_PYTHONCLASS_H +#define CC_MODEL_PYTHONCLASS_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonClass : PythonEntity +{ + std::string toString() const + { + return std::string("PythonClass") + .append("\nid = ").append(std::to_string(id)) + .append("\nqualifiedName = ").append(qualifiedName); + } +}; + +typedef std::shared_ptr PythonClassPtr; + +#pragma db object +struct PythonClassMember +{ + enum Kind + { + Attribute, + Method, + Class + }; + + #pragma db id auto + int id; + + #pragma db unique + PythonAstNodeId astNodeId; + + PythonEntityId memberId; + PythonEntityId classId; + + Kind kind; + bool staticMember = false; + + std::string toString() const + { + return std::string("PythonClassMember") + .append("\nid = ").append(std::to_string(id)) + .append("\nmemberId = ").append(std::to_string(memberId)) + .append("\nclassId = ").append(std::to_string(classId)) + .append("\nstaticMember = ").append(std::to_string(staticMember)) + .append("\nkind = ").append(kind == Attribute ? "Attribute" : + kind == Method ? "Method" : "Class"); + } +}; + +typedef std::shared_ptr PythonClassMemberPtr; + +#pragma db view object(PythonClass) +struct PythonClassCount +{ + #pragma db column("count(" + PythonClass::id + ")") + std::size_t count; +}; + +#pragma db view object(PythonClassMember) +struct PythonClassMemberCount +{ + #pragma db column("count(" + PythonClassMember::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythondocumentation.h b/plugins/python/model/include/model/pythondocumentation.h new file mode 100644 index 000000000..a2900aee7 --- /dev/null +++ b/plugins/python/model/include/model/pythondocumentation.h @@ -0,0 +1,47 @@ +#ifndef CC_MODULE_PYTHONDOCUMENTATION_H +#define CC_MODULE_PYTHONDOCUMENTATION_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonDocumentation +{ + enum Kind + { + Function, + Class, + Module + }; + + #pragma db id auto + int id; + + std::string documentation; + + PythonEntityId documented; + + Kind documentationKind; + + std::string toString() const + { + return std::string("PythonPythonDocumentation") + .append("\nid = ").append(std::to_string(id)) + .append("\ndocumentation = ").append(documentation) + .append("\ndocumented = ").append(std::to_string(documented)) + .append("\ndocumentationKind = ").append( + documentationKind == Function ? "Function" : + documentationKind == Class ? "Class" : "Module"); + } +}; + +typedef std::shared_ptr PythonDocumentationPtr; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonentity.h b/plugins/python/model/include/model/pythonentity.h new file mode 100644 index 000000000..c9c7bbd13 --- /dev/null +++ b/plugins/python/model/include/model/pythonentity.h @@ -0,0 +1,50 @@ +#ifndef CC_MODEL_PYTHONENTITY_H +#define CC_MODEL_PYTHONENTITY_H + +#include + +#include "pythonastnode.h" + +#include + +namespace cc +{ +namespace model +{ + +typedef std::uint64_t PythonEntityId; + +#pragma db object polymorphic +struct PythonEntity +{ + virtual ~PythonEntity() {} + + #pragma db id + PythonEntityId id; + + #pragma db unique + PythonAstNodeId astNodeId; + + std::string name; + std::string qualifiedName; + std::string visibility; +}; + +typedef std::shared_ptr PythonEntityPtr; + +inline PythonEntityId createIdentifier(const PythonEntity& entity_) +{ + std::string res; + + res + .append(std::to_string(entity_.astNodeId)).append(":") + .append(entity_.name).append(":") + .append(entity_.qualifiedName).append(":"); + + return util::fnvHash(res); +} + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonfunction.h b/plugins/python/model/include/model/pythonfunction.h new file mode 100644 index 000000000..0095d3a6e --- /dev/null +++ b/plugins/python/model/include/model/pythonfunction.h @@ -0,0 +1,51 @@ +#ifndef CC_MODEL_PYTHONFUNCTION_H +#define CC_MODEL_PYTHONFUNCTION_H + +#include + +#include "pythonvariable.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonFunction : PythonEntity +{ + #pragma db on_delete(cascade) + std::vector> parameters; + + #pragma db on_delete(cascade) + std::vector> locals; + + std::string toString() const + { + return std::string("PythonFunction") + .append("\nid = ").append(std::to_string(id)) + .append("\nqualifiedName = ").append(qualifiedName); + } +}; + +typedef std::shared_ptr PythonFunctionPtr; + +#pragma db view \ + object(PythonFunction) object(PythonVariable = Parameters : PythonFunction::parameters) +struct PythonFunctionParamCount +{ + #pragma db column("count(" + Parameters::id + ")") + std::size_t count; +}; + +#pragma db view \ + object(PythonFunction) object(PythonVariable = Locals : PythonFunction::locals) +struct PythonFunctionLocalCount +{ + #pragma db column("count(" + Locals::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonimport.h b/plugins/python/model/include/model/pythonimport.h new file mode 100644 index 000000000..bde220e5f --- /dev/null +++ b/plugins/python/model/include/model/pythonimport.h @@ -0,0 +1,61 @@ +#ifndef CC_MODEL_PYTHONIMPORT_H +#define CC_MODEL_PYTHONIMPORT_H + +#include + +#include + +#include + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonImport +{ + #pragma db id auto + int id; + + #pragma db unique + PythonAstNodeId astNodeId; + + #pragma db not_null + #pragma db on_delete(cascade) + odb::lazy_shared_ptr importer; + + #pragma db not_null + #pragma db on_delete(cascade) + odb::lazy_shared_ptr imported; + + PythonEntityId importedSymbol; + + std::string toString() const + { + return std::string("PythonModuleImport") + .append("\nid = ").append(std::to_string(id)) + .append("\nimporter = ").append(std::to_string(importer->id)) + .append("\nimported = ").append(std::to_string(imported->id)) + .append("\nimported symbols =").append(std::to_string(importedSymbol)); + } + +#pragma db index member(importer) +#pragma db index member(imported) +}; + +typedef std::shared_ptr PythonImportPtr; + +#pragma db view object(PythonImport) +struct PythonImportCount +{ + #pragma db column("count(" + PythonImport::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythoninheritance.h b/plugins/python/model/include/model/pythoninheritance.h new file mode 100644 index 000000000..c4713bbfb --- /dev/null +++ b/plugins/python/model/include/model/pythoninheritance.h @@ -0,0 +1,44 @@ +#ifndef CC_MODEL_PYTHONINHERITANCE_H +#define CC_MODEL_PYTHONINHERITANCE_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonInheritance +{ + #pragma db id auto + int id; + + PythonEntityId derived; + PythonEntityId base; + + std::string toString() const + { + return std::string("PythonInheritance") + .append("\nid = ").append(std::to_string(id)) + .append("\nderived = ").append(std::to_string(derived)) + .append("\nbase = ").append(std::to_string(base)); + } + +#pragma db index member(derived) +#pragma db index member(base) +}; + +typedef std::shared_ptr PythonInheritancePtr; + +#pragma db view object(PythonInheritance) +struct PythonInheritanceCount +{ + #pragma db column("count(" + PythonInheritance::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythontype.h b/plugins/python/model/include/model/pythontype.h new file mode 100644 index 000000000..e9e598874 --- /dev/null +++ b/plugins/python/model/include/model/pythontype.h @@ -0,0 +1,34 @@ +#ifndef CC_MODEL_PYTHONTYPE_H +#define CC_MODEL_PYTHONTYPE_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonType +{ + #pragma db id auto + std::uint64_t id; + + PythonEntityId type; + PythonEntityId symbol; + + std::string toString() const + { + return std::string("PythonType") + .append("\nid = ").append(std::to_string(id)) + .append("\ntype = ").append(std::to_string(type)) + .append("\nsymbol = ").append(std::to_string(symbol)); + } +}; + +typedef std::shared_ptr PythonTypePtr; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonvariable.h b/plugins/python/model/include/model/pythonvariable.h new file mode 100644 index 000000000..0a745d395 --- /dev/null +++ b/plugins/python/model/include/model/pythonvariable.h @@ -0,0 +1,27 @@ +#ifndef CC_MODEL_PYTHONVARIABLE_H +#define CC_MODEL_PYTHONVARIABLE_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonVariable : PythonEntity +{ + std::string toString() const + { + return std::string("PythonVariable") + .append("\nid = ").append(std::to_string(id)) + .append("\nqualifiedName = ").append(qualifiedName); + } +}; + +typedef std::shared_ptr PythonVariablePtr; + +} +} + +#endif diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt new file mode 100644 index 000000000..b521ad072 --- /dev/null +++ b/plugins/python/parser/CMakeLists.txt @@ -0,0 +1,32 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) +find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread python) + +include_directories( + include + ${CMAKE_SOURCE_DIR}/util/include + ${CMAKE_SOURCE_DIR}/model/include + ${CMAKE_SOURCE_DIR}/parser/include + ${PLUGIN_DIR}/model/include + ${Python3_INCLUDE_DIRS}) + +add_library(pythonparser SHARED + src/pythonparser.cpp + src/pythonpersistence.cpp) + +target_link_libraries(pythonparser + util + model + pythonmodel + ${Python3_LIBRARIES} + ${Boost_LIBRARIES}) + +target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) +target_link_options(pythonparser PUBLIC -Xlinker -export-dynamic) + +install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) +install( + DIRECTORY ${PLUGIN_DIR}/parser/src/scripts/ + DESTINATION ${INSTALL_PARSER_DIR}/scripts/python + FILES_MATCHING + PATTERN "*.py" + PATTERN "skip.json") diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h new file mode 100644 index 000000000..d2b36fafb --- /dev/null +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -0,0 +1,29 @@ +#ifndef CODECOMPASS_PYTHONPARSER_H +#define CODECOMPASS_PYTHONPARSER_H + + +#include +#include + +namespace cc +{ +namespace parser +{ + +class PythonParser : public AbstractParser { +public: + PythonParser(ParserContext &ctx_); + + virtual ~PythonParser(); + + virtual void markModifiedFiles() override; + + virtual bool cleanupDatabase() override; + + virtual bool parse() override; +}; + +} // parser +} // cc + +#endif //CODECOMPASS_PYTHONPARSER_H diff --git a/plugins/python/parser/include/pythonparser/pythonpersistence.h b/plugins/python/parser/include/pythonparser/pythonpersistence.h new file mode 100644 index 000000000..adff61a87 --- /dev/null +++ b/plugins/python/parser/include/pythonparser/pythonpersistence.h @@ -0,0 +1,121 @@ +#ifndef CODECOMPASS_PYTHONPERSISTENCE_H +#define CODECOMPASS_PYTHONPERSISTENCE_H + +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cc { +namespace parser { + +class PythonPersistence { +private: + struct AstNodeEquality { + public: + bool operator()(const model::PythonAstNodePtr& first, const model::PythonAstNodePtr& second) const { + return first->id == second->id; + } + }; + + struct AstNodeHash { + public: + size_t operator()(const model::PythonAstNodePtr& node) const { + return node->id; + } + }; + + using AstNodeSet = std::unordered_set; + + ParserContext &ctx; + +public: + PythonPersistence(ParserContext &ctx_) : ctx(ctx_) {} + + ~PythonPersistence(); + + void cppprint(boost::python::object o); + + void logInfo(boost::python::object o); + void logWarning(boost::python::object o); + void logError(boost::python::object o); + void logDebug(boost::python::object o); + + void persistFile(boost::python::object pyFile); + + void persistVariable(boost::python::object pyVariable); + + void persistFunction(boost::python::object pyFunction); + + void persistPreprocessedClass(boost::python::object pyClass); + + void persistClass(boost::python::object pyClass); + + void persistImport(boost::python::object pyImport); + +private: + boost::optional createFileLoc(boost::python::object filePosition); + + model::PythonEntityPtr getPythonEntity(const std::string &qualifiedName); + + model::PythonClassPtr getPythonClass(const std::string &qualifiedName); + + model::PythonVariablePtr getPythonVariable(const std::string &qualifiedName); + + bool isAstNodePersisted(const model::PythonAstNodePtr &node) const; + + bool isAstNodePersisted(const std::vector &nodes, + const model::PythonAstNodePtr &node) const; + + AstNodeSet _astNodes; + std::vector _variables; + std::map _variableUsages; + std::vector _functions; + std::map _functionUsages; + std::vector _classes; + std::map _classUsages; + std::vector _members; + std::vector _inheritance; + std::vector _imports; + std::vector _documentations; + std::vector _types; +}; + +typedef boost::shared_ptr PythonPersistencePtr; + +} +} + +#endif //CODECOMPASS_PYTHONPERSISTENCE_H diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp new file mode 100644 index 000000000..a23b03ee8 --- /dev/null +++ b/plugins/python/parser/src/pythonparser.cpp @@ -0,0 +1,117 @@ +#include +#include + +#include +#include +#include + +#include + +#include + +#include + +namespace cc { +namespace parser{ + +BOOST_PYTHON_MODULE(persistence) +{ + boost::python::class_("Persistence", boost::python::init()) + .def("print", &PythonPersistence::cppprint) + .def("log_info", &PythonPersistence::logInfo) + .def("log_warning", &PythonPersistence::logWarning) + .def("log_error", &PythonPersistence::logError) + .def("log_debug", &PythonPersistence::logDebug) + .def("persist_file", &PythonPersistence::persistFile) + .def("persist_variable", &PythonPersistence::persistVariable) + .def("persist_function", &PythonPersistence::persistFunction) + .def("persist_preprocessed_class", &PythonPersistence::persistPreprocessedClass) + .def("persist_class", &PythonPersistence::persistClass) + .def("persist_import", &PythonPersistence::persistImport); +} + +PythonParser::PythonParser(ParserContext &ctx_) : AbstractParser(ctx_) {} + +PythonParser::~PythonParser() +{ +} + +void PythonParser::markModifiedFiles() {} + +bool PythonParser::cleanupDatabase() { return true; } + +bool PythonParser::parse() +{ + const std::string PARSER_SCRIPTS_DIR = _ctx.compassRoot + "/lib/parserplugin/scripts/python"; + if (!boost::filesystem::exists(PARSER_SCRIPTS_DIR) || !boost::filesystem::is_directory(PARSER_SCRIPTS_DIR)) + { + throw std::runtime_error(PARSER_SCRIPTS_DIR + " is not a directory!"); + } + + setenv("PYTHONPATH", PARSER_SCRIPTS_DIR.c_str(), 1); + + try{ + Py_Initialize(); + init_module_persistence(); + + boost::python::object module = boost::python::import("cc_python_parser.python_parser"); + + if(!module.is_none()) + { + boost::python::object func = module.attr("parse"); + + if(!func.is_none() && PyCallable_Check(func.ptr())) + { + std::string source_path; + for (const std::string& input : _ctx.options["input"].as>()) + { + if (boost::filesystem::is_directory(input)) + { + source_path = input; + } + } + if(source_path.empty()) + { + LOG(error) << "No source path was found"; + } else { + PythonPersistencePtr persistencePtr(new PythonPersistence(_ctx)); + + func(source_path, boost::python::ptr(persistencePtr.get())); + } + } else { + LOG(error) << "Cannot find function"; + } + } else { + LOG(error) << "Cannot import module"; + } + } catch (boost::python::error_already_set) + { + PyErr_Print(); + } + + return true; +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ +boost::program_options::options_description getOptions() +{ + boost::program_options::options_description description("Python Plugin"); + description.add_options() + ("skip-doccomment", + "If this flag is given the parser will skip parsing the documentation " + "comments."); + return description; +} + +std::shared_ptr make(ParserContext& ctx_) +{ + return std::make_shared(ctx_); +} +} +#pragma clang diagnostic pop + +} +} diff --git a/plugins/python/parser/src/pythonpersistence.cpp b/plugins/python/parser/src/pythonpersistence.cpp new file mode 100644 index 000000000..525630557 --- /dev/null +++ b/plugins/python/parser/src/pythonpersistence.cpp @@ -0,0 +1,959 @@ +#include + +namespace cc +{ +namespace parser +{ + +using boost::python::object; +using boost::python::len; +using boost::python::tuple; +using boost::python::list; +using boost::python::dict; + +template +using extract = boost::python::extract; + +void print(const std::string& s) +{ + LOG(info) << s; +} + +template +static std::vector getKeys(const std::map& map) +{ + std::vector ret; + std::for_each(map.begin(), map.end(), [&ret](const auto& p) + { ret.push_back(p.first); }); + return ret; +} + +PythonPersistence::~PythonPersistence() +{ + std::map m; + std::vector v = getKeys(m); + ctx.srcMgr.persistFiles(); + + (util::OdbTransaction(ctx.db))([this]{ + std::vector v = std::vector(_astNodes.begin(), _astNodes.end()); + util::persistAll(v, ctx.db); + for(auto& ast : _variableUsages) + { + std::vector u = std::vector(ast.second.begin(), ast.second.end()); + util::persistAll(u, ctx.db); + } + for(auto& ast : _functionUsages) + { + std::vector u = std::vector(ast.second.begin(), ast.second.end()); + util::persistAll(u, ctx.db); + } + for(auto& ast : _classUsages) + { + std::vector u = std::vector(ast.second.begin(), ast.second.end()); + util::persistAll(u, ctx.db); + } + util::persistAll(_variables, ctx.db); + util::persistAll(_functions, ctx.db); + util::persistAll(_classes, ctx.db); + util::persistAll(_members, ctx.db); + util::persistAll(_inheritance, ctx.db); + util::persistAll(_imports, ctx.db); + util::persistAll(_documentations, ctx.db); + util::persistAll(_types, ctx.db); + }); +} + +void PythonPersistence::cppprint(object o) +{ + std::string s = extract(o); + LOG(info) << s; +} + +void PythonPersistence::logInfo(object o) +{ + std::string s = extract(o); + LOG(info) << s; +} + +void PythonPersistence::logWarning(object o) +{ + std::string s = extract(o); + LOG(warning) << s; +} + +void PythonPersistence::logError(object o) +{ + std::string s = extract(o); + LOG(error) << s; +} + +void PythonPersistence::logDebug(object o) +{ + std::string s = extract(o); + LOG(debug) << s; +} + +void PythonPersistence::persistFile(object pyFile) +{ + try{ + model::FilePtr file = nullptr; + model::BuildSource buildSource; + + object path = pyFile.attr("path"); + object status = pyFile.attr("parse_status"); + + if(status.is_none()) + { + LOG(error) << "Persist file: status is None..."; + } else if(path.is_none()) + { + LOG(error) << "Persist file: path is None..."; + } else { + file = ctx.srcMgr.getFile(extract(path)); + file->type = "PY"; + buildSource.file = file; + switch(extract(status)) + { + case 0: + buildSource.file->parseStatus = model::File::PSNone; + break; + case 1: + buildSource.file->parseStatus = model::File::PSPartiallyParsed; + break; + case 2: + buildSource.file->parseStatus = model::File::PSFullyParsed; + break; + default: + LOG(error) << "Persist file unknown status: " << extract(status); + } + + model::BuildActionPtr buildAction(new model::BuildAction); + buildAction->command = ""; + buildAction->type = model::BuildAction::Interpret; + try{ + util::OdbTransaction transaction{ctx.db}; + + transaction([&, this] { + ctx.db->persist(buildAction); + }); + + buildSource.action = buildAction; + + ctx.srcMgr.updateFile(*buildSource.file); + + transaction([&, this] { + ctx.db->persist(buildSource); + }); + } catch(const std::exception& ex) + { + LOG(error) << "Persist file exception: " << ex.what() << " - " << typeid(ex).name(); + } + } + } catch(std::exception e) + { + LOG(error) << "Persist file exception: " << e.what(); + } +} + +void PythonPersistence::persistVariable(object pyVariable) +{ + try{ + object name = pyVariable.attr("name"); + object qualifiedName = pyVariable.attr("qualified_name"); + object visibility = pyVariable.attr("visibility"); + + boost::optional fileLoc = + createFileLoc(pyVariable.attr("file_position")); + + list types = + extract(pyVariable.attr("type")); + + list usages = + extract(pyVariable.attr("usages")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + types.is_none() || usages.is_none()) + { + return; + } + + model::PythonAstNodePtr varAstNode(new model::PythonAstNode); + varAstNode->location = fileLoc.get(); + varAstNode->qualifiedName = extract(qualifiedName); + varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + varAstNode->astType = model::PythonAstNode::AstType::Declaration; + + varAstNode->id = model::createIdentifier(*varAstNode); + + if(!_astNodes.insert(varAstNode).second) + { + return; + } + + model::PythonVariablePtr variable(new model::PythonVariable); + variable->astNodeId = varAstNode->id; + variable->name = extract(name); + variable->qualifiedName = extract(qualifiedName); + variable->visibility = extract(visibility); + + variable->id = model::createIdentifier(*variable); + _variables.push_back(variable); + + for(int i = 0; i(types[i]); + model::PythonEntityPtr t = + getPythonEntity(extract(types[i])); + if(!t) + { + continue; + } + type->type = t->id; + type->symbol = variable->id; + _types.push_back(type); + } + + _variableUsages[variable->id] = {}; + for(int i = 0; i usageFileLoc = + createFileLoc(usages[i].attr("file_position")); + if(usageFileLoc == boost::none) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _variableUsages[variable->id].insert(usageAstNode); + } + + } catch (const odb::object_already_persistent& ex) + { + LOG(error) << "Persist variable exception already persistent: " << ex.what() << std::endl; + } catch (const odb::database_exception& ex) + { + LOG(error) << "Persist variable exception db exception: " << ex.what(); + } catch(std::exception ex) + { + LOG(error) << "Persist variable exception: " << ex.what(); + } +} + +void PythonPersistence::persistFunction(object pyFunction) +{ + try{ + object name = pyFunction.attr("name"); + object qualifiedName = pyFunction.attr("qualified_name"); + object visibility = pyFunction.attr("visibility"); + + boost::optional fileLoc = + createFileLoc(pyFunction.attr("file_position")); + + list types = + extract(pyFunction.attr("type")); + + list usages = + extract(pyFunction.attr("usages")); + + object pyDocumentation = pyFunction.attr("documentation"); + + list params = + extract(pyFunction.attr("parameters")); + + list locals = + extract(pyFunction.attr("locals")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || types.is_none() || + usages.is_none() || pyDocumentation.is_none() || params.is_none() || locals.is_none()) + { + return; + } + + model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); + funcAstNode->location = fileLoc.get(); + funcAstNode->qualifiedName = extract(qualifiedName); + funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + funcAstNode->astType = model::PythonAstNode::AstType::Declaration; + + funcAstNode->id = model::createIdentifier(*funcAstNode); + + if(!_astNodes.insert(funcAstNode).second) + { + return; + } + + model::PythonFunctionPtr function(new model::PythonFunction); + function->astNodeId = funcAstNode->id; + function->name = extract(name); + function->qualifiedName = extract(qualifiedName); + function->visibility = extract(visibility); + + function->id = model::createIdentifier(*function); + + for(int i = 0; i(params[i])); + if(!param) + { + continue; + } + function->parameters.push_back(param); + } + + for(int i = 0; i(locals[i])); + if(!local) + { + continue; + } + function->locals.push_back(local); + } + + _functions.push_back(function); + + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = extract(pyDocumentation); + documentation->documented = function->id; + documentation->documentationKind = model::PythonDocumentation::Function; + + _documentations.push_back(documentation); + + for(int i = 0; i(types[i])); + if(!t) + { + continue; + } + type->type = t->id; + type->symbol = function->id; + + _types.push_back(type); + } + + _functionUsages[function->id] = {}; + for(int i = 0; i usageFileLoc = + createFileLoc(usages[i].attr("file_position")); + if(usageFileLoc == boost::none) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _functionUsages[function->id].insert(usageAstNode); + } + } catch(std::exception e) + { + LOG(error) << "Persist function exception:" << e.what(); + } +} + +void PythonPersistence::persistPreprocessedClass(object pyClass) +{ + try{ + object name = pyClass.attr("name"); + object qualifiedName = pyClass.attr("qualified_name"); + object visibility = pyClass.attr("visibility"); + + boost::optional fileLoc = + createFileLoc(pyClass.attr("file_position")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none) + { + return; + } + + model::PythonAstNodePtr classAstNode(new model::PythonAstNode); + classAstNode->location = fileLoc.get(); + classAstNode->qualifiedName = extract(qualifiedName); + classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + classAstNode->astType = model::PythonAstNode::AstType::Declaration; + + classAstNode->id = model::createIdentifier(*classAstNode); + + if(!_astNodes.insert(classAstNode).second) + { + return; + } + + model::PythonClassPtr cl(new model::PythonClass); + cl->astNodeId = classAstNode->id; + cl->name = extract(name); + cl->qualifiedName = extract(qualifiedName); + cl->visibility = extract(visibility); + + cl->id = model::createIdentifier(*cl); + + _classes.push_back(cl); + + } catch(std::exception e) + { + LOG(error) << "Persist preprocessed class exception:" << e.what(); + } +} + +void PythonPersistence::persistClass(object pyClass) +{ + try{ + object qualifiedName = pyClass.attr("qualified_name"); + + list usages = + extract(pyClass.attr("usages")); + + object pyDocumentation = pyClass.attr("documentation"); + + list baseClasses = + extract(pyClass.attr("base_classes")); + + object members = pyClass.attr("members"); + + if(qualifiedName.is_none() || usages.is_none() || pyDocumentation.is_none() || + baseClasses.is_none() || members.is_none()) + { + return; + } + + model::PythonClassPtr cl = + getPythonClass(extract(qualifiedName)); + + if (!cl) + { + LOG(error) << "Persist class: cl is none"; + return; + } + + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = extract(pyDocumentation); + documentation->documented = cl->id; + documentation->documentationKind = model::PythonDocumentation::Class; + + _documentations.push_back(documentation); + + _classUsages[cl->id] = {}; + for(int i = 0; i usageFileLoc = + createFileLoc(usages[i].attr("file_position")); + if(usageFileLoc == boost::none) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _classUsages[cl->id].insert(usageAstNode); + } + + for(int i = 0; iderived = cl->id; + std::string baseClassQualifiedName = extract(baseClasses[i]); + + inheritance->base = getPythonEntity(baseClassQualifiedName)->id; + + _inheritance.push_back(inheritance); + } + + list methods = + extract(members.attr("methods")); + list staticMethods = + extract(members.attr("static_methods")); + list attributes = + extract(members.attr("attributes")); + list staticAttributes = + extract(members.attr("static_attributes")); + list classes = + extract(members.attr("classes")); + + if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || + staticAttributes.is_none() || classes.is_none()) + { + return; + } + + for(int i = 0; i(methods[i].attr("qualified_name")); + model::PythonEntityPtr method = getPythonEntity(qualifiedName); + if(!method) + { + LOG(error) << "Persist class: method is none"; + continue; + } + classMember->astNodeId = method->astNodeId; + classMember->memberId = method->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = false; + + _members.push_back(classMember); + + list _usages = + extract(methods[i].attr("usages")); + AstNodeSet& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + }) != _funcUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _funcUsages.insert(usageAstNode); + } + } + + for(int i = 0; i(staticMethods[i].attr("qualified_name")); + model::PythonEntityPtr method = getPythonEntity(qualifiedName); + if(!method) + { + LOG(error) << "Persist class: static method is none"; + continue; + } + classMember->astNodeId = method->astNodeId; + classMember->memberId = method->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = true; + + _members.push_back(classMember); + + list _usages = + extract(staticMethods[i].attr("usages")); + AstNodeSet& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if (_funcUsages.begin(), _funcUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + } + ) != _funcUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _funcUsages.insert(usageAstNode); + } + } + + for(int i = 0; i(attributes[i].attr("qualified_name")); + if (qualifiedName.empty()) + { + continue; // TODO: import symbol in class + } + model::PythonEntityPtr attr = getPythonEntity(qualifiedName); + if(!attr) + { + LOG(error) << "Persist class: attr is none"; + continue; + } + classMember->astNodeId = attr->astNodeId; + classMember->memberId = attr->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = false; + + _members.push_back(classMember); + + list _usages = + extract(attributes[i].attr("usages")); + AstNodeSet& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + }) != _varUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _varUsages.insert(usageAstNode); + } + } + + for(int i = 0; i(staticAttributes[i].attr("qualified_name")); + model::PythonEntityPtr attr = getPythonEntity(qualifiedName); + if(!attr) + { + LOG(error) << "Persist class: static attr is none"; + continue; + } + classMember->astNodeId = attr->astNodeId; + classMember->memberId = attr->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = true; + + _members.push_back(classMember); + + list _usages = + extract(staticAttributes[i].attr("usages")); + AstNodeSet& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + }) != _varUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _varUsages.insert(usageAstNode); + } + } + + for(int i = 0; i(classes[i].attr("qualified_name")); + model::PythonEntityPtr inner = getPythonEntity(qualifiedName); + if(!inner) + { + LOG(error) << "Persist class: inner class is none"; + continue; + } + classMember->astNodeId = inner->astNodeId; + classMember->memberId = inner->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Class; + classMember->staticMember = false; + + _members.push_back(classMember); + } + } catch(std::exception e) + { + LOG(error) << "Persist class exception:" << e.what(); + } +} + +void PythonPersistence::persistImport(object pyImport) +{ + try { + model::FilePtr file = + ctx.srcMgr.getFile(extract(pyImport.attr("importer"))); + + list importedModules = + extract(pyImport.attr("imported_modules")); + + dict importedSymbols = + extract(pyImport.attr("imported_symbols")); + + if (!file || importedModules.is_none() || importedSymbols.is_none()) + { + return; + } + + for (int i = 0; i < len(importedModules); ++i) + { + object importData = importedModules[i]; + + model::FilePtr moduleFile = + ctx.srcMgr.getFile(extract(importData.attr("imported"))); + boost::optional fileLoc = createFileLoc(importData.attr("position")); + std::string qualifiedName = extract(importData.attr("qualified_name")); + + if (!moduleFile || fileLoc == boost::none) + { + continue; + } + + model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); + moduleAstNode->location = fileLoc.get(); + moduleAstNode->qualifiedName = qualifiedName; + moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; + moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; + + moduleAstNode->id = model::createIdentifier(*moduleAstNode); + + if(!_astNodes.insert(moduleAstNode).second) + { + continue; + } + + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->astNodeId = moduleAstNode->id; + moduleImport->importer = file; + moduleImport->imported = moduleFile; + + _imports.push_back(moduleImport); + } + + list importDict = importedSymbols.items(); + for (int i = 0; i < len(importDict); ++i) + { + tuple import = extract(importDict[i]); + + object importData = import[0]; + + model::FilePtr moduleFile = + ctx.srcMgr.getFile(extract(importData.attr("imported"))); + boost::optional fileLoc = createFileLoc(importData.attr("position")); + std::string qualifiedName = extract(importData.attr("qualified_name")); + + if (!moduleFile || fileLoc == boost::none) + { + continue; + } + + model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); + moduleAstNode->location = fileLoc.get(); + moduleAstNode->qualifiedName = qualifiedName; + moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; + moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; + + moduleAstNode->id = model::createIdentifier(*moduleAstNode); + + if(!_astNodes.insert(moduleAstNode).second) + { + continue; + } + + for (int j = 0; j < len(import[1]); ++j) + { + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->astNodeId = moduleAstNode->id; + moduleImport->importer = file; + moduleImport->imported = moduleFile; + auto symb = + getPythonEntity(extract(import[1][j])); + if(!symb) + { + continue; + } + moduleImport->importedSymbol = symb->id; + + _imports.push_back(moduleImport); + } + } + } catch(std::exception e) + { + LOG(error) << "Persist import exception:" << e.what(); + } +} + +boost::optional PythonPersistence::createFileLoc(object filePosition) +{ + if (filePosition.is_none()) + { + return boost::none; + } + + object filePath = filePosition.attr("file"); + object pyRange = filePosition.attr("range"); + + if (filePath.is_none() || pyRange.is_none()) + { + return boost::none; + } + + object pyStartPosition = pyRange.attr("start_position"); + object pyEndPosition = pyRange.attr("end_position"); + + if (pyStartPosition.is_none() || pyEndPosition.is_none()) + { + return boost::none; + } + + model::FileLoc fileLoc; + + fileLoc.file = ctx.srcMgr.getFile(extract(filePath)); + + model::Position startPosition(extract(pyStartPosition.attr("line")), + extract(pyStartPosition.attr("column"))); + model::Position endPosition(extract(pyEndPosition.attr("line")), + extract(pyEndPosition.attr("column"))); + + fileLoc.range = model::Range(startPosition, endPosition); + + return fileLoc; +} + +model::PythonEntityPtr PythonPersistence::getPythonEntity(const std::string& qualifiedName) +{ + if(qualifiedName.empty()) + { + return nullptr; + } + + auto varIt = + std::find_if(_variables.begin(), + _variables.end(), + [&](const auto& var) + { return var->qualifiedName == qualifiedName; } + ); + if (varIt != _variables.end()) + { + return *varIt; + } + + auto funcIt = + std::find_if(_functions.begin(), + _functions.end(), + [&](const auto& func) + { return func->qualifiedName == qualifiedName; } + ); + if (funcIt != _functions.end()) + { + return *funcIt; + } + + auto classIt = + std::find_if(_classes.begin(), + _classes.end(), + [&](const auto& cl) + { return cl->qualifiedName == qualifiedName; } + ); + if (classIt != _classes.end()) + { + return *classIt; + } + + return nullptr; +} + +model::PythonVariablePtr PythonPersistence::getPythonVariable(const std::string& qualifiedName) +{ + if(qualifiedName.empty()) + { + return nullptr; + } + + auto varIt = + std::find_if(_variables.begin(), + _variables.end(), + [&](const auto& var) + { return var->qualifiedName == qualifiedName; } + ); + if (varIt != _variables.end()) + { + return *varIt; + } + + return nullptr; +} + +model::PythonClassPtr PythonPersistence::getPythonClass(const std::string& qualifiedName) +{ + auto classIt = + std::find_if(_classes.begin(), + _classes.end(), + [&](const auto& cl) + { return cl->qualifiedName == qualifiedName; } + ); + if (classIt != _classes.end()) + { + return *classIt; + } + + return nullptr; +} + +//bool PythonPersistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const +//{ +// for(const auto &it : _astNodes) +// { +// if(*it == *node) +// { +// return true; +// } +// } +// +// return false; +//} + +//bool PythonPersistence::isAstNodePersisted( +// const std::vector& nodes, +// const model::PythonAstNodePtr& node) const +//{ +// for(const auto &it : nodes) +// { +// if(*it == *node) +// { +// return true; +// } +// } +// +// return false; +//} + +} +} \ No newline at end of file diff --git a/plugins/python/parser/src/scripts/cc_python_parser/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/__init__.py new file mode 100644 index 000000000..7d7338820 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/__init__.py @@ -0,0 +1,7 @@ +__all__ = ["base_data", "built_in_functions", "built_in_operators", "built_in_types", "class_data", + "class_init_declaration", "class_preprocessor", "file_info", "function_data", "function_symbol_collector", + "function_symbol_collector_factory", "import_finder", "import_preprocessor", "logger", "main", + "member_access_collector", "parser", "parse_exception", "placeholder_function_declaration_cache", + "preprocessed_data", "preprocessed_file", "preprocessed_function", "preprocessed_variable", "python_parser", + "scope", "scope_manager", "symbol_collector", "symbol_collector_interface", "symbol_finder", "type_data", + "type_deduction", "variable_data"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/base_data.py b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py new file mode 100644 index 000000000..da664bd0b --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py @@ -0,0 +1,74 @@ +from typing import List, TypeVar, Generic, Set, Optional + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.type_data import DeclarationType + + +class Usage: + def __init__(self, name: str, position: FilePosition): + self.name = name + self.file_position = position + + def __str__(self): + return self.name + ' (' + str(self.file_position) + ')' + + def __repr__(self): + return self.__str__() + + +class Declaration(DeclarationType): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, + declaration_type: Optional[Set[DeclarationType]] = None): + DeclarationType.__init__(self, name, qualified_name) + assert declaration_type is None or not any(isinstance(x, type) for x in declaration_type) + self.file_position: FilePosition = pos + if declaration_type is None: + self.type: Set[DeclarationType] = set() + else: + self.type: Set[DeclarationType] = declaration_type + self.usages: List[Usage] = [] + + def is_same_declaration(self, other: 'Declaration') -> bool: + return self.__eq__(other) and self.file_position == other.file_position + + def is_same_usage(self, other: Usage): + return self.name == other.name and self.file_position == other.file_position + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.qualified_name) ^ hash(self.file_position) + + def get_type_repr(self) -> str: + return '[' + ','.join({x.get_type_repr() if x is not None else 'None' for x in self.type}) + ']' + + +class TypeDeclaration(Declaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition): + super().__init__(name, qualified_name, pos) + self.type.add(self) + + +T = TypeVar('T', bound=Declaration) + + +class ImportedDeclaration(Generic[T]): + def __init__(self, qualified_name: str, declaration: T, module, pos: FilePosition): + self.qualified_name = qualified_name + self.imported_declaration = declaration + self.module = module + self.position = pos + + +class ReferenceDeclaration: + def __init__(self, reference: Declaration): + self.reference: Declaration = reference + + +class DocumentedType: + def __init__(self, documentation: str): + self.documentation: str = documentation diff --git a/plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py b/plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py new file mode 100644 index 000000000..064c78788 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py @@ -0,0 +1,616 @@ +import inspect +import sys +from abc import ABC, abstractmethod +from typing import Optional, Set, Union, List + +from cc_python_parser import built_in_types +from cc_python_parser.built_in_types import Boolean, Complex, Dictionary, Float, FrozenSet, Integer, File, Slice, Tuple, \ + Object, MemoryView, String, Bytes, ByteArray, Type, RangeType +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.type_data import DeclarationType + + +class BuiltInFunction(FunctionDeclaration, ABC): + def __init__(self): + file_position = FilePosition.get_empty_file_position() + qualified_name = "builtins.function." + self.get_name() + super().__init__(self.get_name(), qualified_name, file_position, [], "", self.get_type()) + if self.get_override() is None: + self.override = [] + else: + self.override: List[str] = self.get_override() + + @staticmethod + @abstractmethod + def get_name() -> str: + pass + + @staticmethod + def get_override() -> Optional[List[str]]: + return None + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return None + + def get_type_repr(self) -> str: + if self.get_type() is None: + return "" + return ','.join([x.get_type_repr() for x in self.get_type()]) + + +class FunctionDecorator(BuiltInFunction, ABC): + pass + + +class AbsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'abs' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__abs__'] + + +class AllFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'all' + + +class AnyFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'any' + + +class AsciiFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'ascii' + + +# returns binary string +class BinFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bin' + + +class BoolFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bool' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class BreakpointFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'breakpoint' + + +class BytearrayFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bytearray' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {ByteArray()} + + +class BytesFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bytes' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Bytes()} + + +class CallableFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'callable' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class ChrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'chr' + + +class ClassmethodFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'classmethod' + + +class CompileFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'compile' + + +class ComplexFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'complex' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__complex__', '__float__', '__index__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Complex()} + + +class DelattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'delattr' + + +class DictFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'dict' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Dictionary()} + + +class DirFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'dir' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__dir__'] + + +class DivmodFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'divmod' + + +# returns enumerate object +class EnumerateFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'enumerate' + + +class EvalFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'eval' + + +class ExecFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'exec' + + +class FilterFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'filter' + + +class FloatFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'float' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Float()} + + +class FormatFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'format' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__format__'] + + +class FrozensetFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'frozenset' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {FrozenSet()} + + +class GetattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'getattr' + + +class GlobalsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'globals' + + +class HasattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hasattr' + + +class HashFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hash' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__hash__'] + + +class HelpFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'help' + + +class HexFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hex' + + +class IdFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'id' + + +class InputFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'input' + + +class IntFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'int' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__init__', '__index__', '__truncate__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Integer()} + + +class IsinstanceFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'isinstance' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class IssubclassFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'issubclass' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +# returns Iterator +class IterFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'iter' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__iter__'] + + +class LenFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'len' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Integer()} + + +class ListFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'list' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {built_in_types.List()} + + +class LocalsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'locals' + + +class MapFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'map' + + +class MaxFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'max' + + +class MemoryViewFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'memoryview' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {MemoryView()} + + +class MinFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'min' + + +class NextFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'next' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__next__'] + + +class ObjectFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'object' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Object()} + + +class OctFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'oct' + + +class OpenFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'open' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {File()} + + +class OrdFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'ord' + + +class PowFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'pow' + + +class PrintFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'print' + + +class PropertyFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'property' + + +class RangeFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'range' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {RangeType()} + + +class ReprFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'repr' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__repr__'] + + +# returns iterator +class ReversedFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'reversed' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__reversed__'] + + +class RoundFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'round' + + +class SetFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'set' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {built_in_types.Set()} + + +class SetattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'setattr' + + +class SliceFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'slice' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Slice()} + + +class SortedFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'sorted' + + +class StaticmethodFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'staticmethod' + + +class StrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'str' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__str__', '__repr__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {String()} + + +class SumFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'sum' + + +class SuperFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'super' + + +class TupleFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'tuple' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Tuple()} + + +class TypeFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'type' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Type()} + + +class VarsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'vars' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__dict__'] + + +class ZipFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'zip' + + +class ImportFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return '__import__' + + +built_in_functions = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltInFunction) and + x is not BuiltInFunction and x is not FunctionDecorator) + + +def get_built_in_function(name: str) -> Optional[BuiltInFunction]: + bif = [x for x in built_in_functions if x[1].get_name() == name] + assert len(bif) <= 1 + if len(bif) == 0: + return None + + return bif[0][1]() + + +def get_all_built_in_functions() -> List[BuiltInFunction]: + return [bif[1]() for bif in built_in_functions] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py b/plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py new file mode 100644 index 000000000..d0bbeb902 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py @@ -0,0 +1,324 @@ +import ast +import inspect +import sys +from abc import ABC, abstractmethod +from typing import Type, List, Optional + +from cc_python_parser.built_in_types import Complex, Float, Integer, Boolean + + +class BuiltInOperator(ABC): + def __init__(self): + self.ast_type = self.get_ast_type() + self.override = self.get_override() + + @staticmethod + @abstractmethod + def get_ast_type() -> Type[ast.AST]: + pass + + @staticmethod + def get_override() -> List[str]: + return [] + + # @staticmethod + # @abstractmethod + # def get_type() -> Type: + # pass + + +class UnaryOperator(BuiltInOperator, ABC): + pass + + +class BinaryOperator(BuiltInOperator, ABC): + pass + + +class CompareOperator(BuiltInOperator, ABC): + pass + + +class BooleanOperator(BuiltInOperator, ABC): + pass + + +class AddOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Add + + @staticmethod + def get_override() -> List[str]: + return ['__add__'] + + +class SubOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Sub + + @staticmethod + def get_override() -> List[str]: + return ['__sub__'] + + +class MultOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Mult + + @staticmethod + def get_override() -> List[str]: + return ['__mul__'] + + +class MatMultOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.MatMult + + @staticmethod + def get_override() -> List[str]: + return ['__matmul__'] + + +class DivOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Div + + @staticmethod + def get_override() -> List[str]: + return ['__truediv__'] + + +class ModOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Mod + + @staticmethod + def get_override() -> List[str]: + return ['__mod__'] + + +class PowOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Pow + + @staticmethod + def get_override() -> List[str]: + return ['__pow__'] + + +class LShiftOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.LShift + + @staticmethod + def get_override() -> List[str]: + return ['__lshift__'] + + +class RShiftOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.RShift + + @staticmethod + def get_override() -> List[str]: + return ['__rshift__'] + + +class BitOrOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitOr + + @staticmethod + def get_override() -> List[str]: + return ['__or__'] + + +class BitXorOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitXor + + @staticmethod + def get_override() -> List[str]: + return ['__xor__'] + + +class BitAndOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitAnd + + @staticmethod + def get_override() -> List[str]: + return ['__and__'] + + +class FloorDivOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.FloorDiv + + @staticmethod + def get_override() -> List[str]: + return ['__floordiv__'] + + +class AndOperator(BooleanOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.And + + +class OrOperator(BooleanOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Or + + +class EqOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Eq + + @staticmethod + def get_override() -> List[str]: + return ['__eq__'] + + +class NotEqOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.NotEq + + @staticmethod + def get_override() -> List[str]: + return ['__ne__'] + + +class LtOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Lt + + @staticmethod + def get_override() -> List[str]: + return ['__lt__'] + + +class LtEOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.LtE + + @staticmethod + def get_override() -> List[str]: + return ['__le__'] + + +class GtOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Gt + + @staticmethod + def get_override() -> List[str]: + return ['__gt__'] + + +class GtEOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.GtE + + @staticmethod + def get_override() -> List[str]: + return ['__ge__'] + + +class IsOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Is + + +class IsNotOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.IsNot + + +class InOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.In + + +class NotInOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.NotIn + + +class InvertOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Invert + + @staticmethod + def get_override() -> List[str]: + return ['__inv__', '__invert__'] + + +class NotOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Not + + @staticmethod + def get_override() -> List[str]: + return ['__not__'] + + +class UAddOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.UAdd + + @staticmethod + def get_override() -> List[str]: + return ['__pos__'] + + +class USubOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.USub + + @staticmethod + def get_override() -> List[str]: + return ['__neg__'] + + +built_in_operators = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltInOperator) and + x is not BuiltInOperator) + +number_precedence = [Complex, Float, Integer, Boolean] + + +def get_built_in_operator(t: ast.AST) -> Optional[BuiltInOperator]: + bio = [x for x in built_in_operators if type(t) is x[1].get_ast_type()] + assert len(bio) <= 1 + if len(bio) == 0: + return None + return bio[0][1]() diff --git a/plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py b/plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py new file mode 100644 index 000000000..30371bfb7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py @@ -0,0 +1,255 @@ +import sys +import typing +import inspect +from abc import ABC, abstractmethod +from typing import Optional, Any + +from cc_python_parser.base_data import TypeDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.class_dto import ClassDeclarationDTO + + +class BuiltIn(TypeDeclaration, ABC): + def __init__(self): + super().__init__(self.get_name(), "builtins.type." + self.get_name(), FilePosition.get_empty_file_position()) + + def get_type_repr(self) -> str: + return self.name + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def create_dto(self) -> ClassDeclarationDTO: + return ClassDeclarationDTO(self.name, self.qualified_name, self.file_position, + set(), [], set(), ClassDeclarationDTO.ClassMembersDTO(), "") + + @staticmethod + @abstractmethod + def get_name() -> str: + pass + + +class GenericType: + def __init__(self, types: Optional[typing.Set[Any]] = None): + if types is None: + self.types = set() + else: + self.types = types + + def add_type(self, new_type): + self.types.add(new_type) + + +class GenericBuiltInType(BuiltIn, GenericType, ABC): + def __init__(self, types: Optional[typing.Set[Any]] = None): + BuiltIn.__init__(self) + GenericType.__init__(self, types) + + def get_type_repr(self) -> str: + return self.name + '<' + ','.join({x.get_type_repr() for x in self.types}) + '>' + + +class Boolean(BuiltIn): + @staticmethod + def get_name() -> str: + return 'bool' + + +class Integer(BuiltIn): + @staticmethod + def get_name() -> str: + return 'int' + + +class Float(BuiltIn): + @staticmethod + def get_name() -> str: + return 'float' + + +class Complex(BuiltIn): + @staticmethod + def get_name() -> str: + return 'complex' + + +class String(BuiltIn): + @staticmethod + def get_name() -> str: + return 'str' + + +class List(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'list' + + +class Tuple(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'tuple' + + +class Bytes(BuiltIn): + @staticmethod + def get_name() -> str: + return 'bytes' + + +class ByteArray(BuiltIn): + @staticmethod + def get_name() -> str: + return 'bytearray' + + +class MemoryView(BuiltIn): + @staticmethod + def get_name() -> str: + return 'memoryview' + + +class Set(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'set' + + +class FrozenSet(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'frozenset' + + +class Dictionary(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'dict' + + +class RangeType(BuiltIn): # generic? (implement __index__ method) + @staticmethod + def get_name() -> str: + return 'range' + + +class NoneType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'None' + + +class EllipsisType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'Ellipsis' + + +class NotImplementedType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'NotImplemented' + + +class Slice(BuiltIn): + @staticmethod + def get_name() -> str: + return 'slice' + + +class Module(BuiltIn): + @staticmethod + def get_name() -> str: + return 'module' + + +class Function(BuiltIn): + @staticmethod + def get_name() -> str: + return 'function' + + +class Method(BuiltIn): + @staticmethod + def get_name() -> str: + return 'method' + + +class Lambda(BuiltIn): + @staticmethod + def get_name() -> str: + return 'lambda' + + +class Generator(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'generator' + + +class Type(BuiltIn): + @staticmethod + def get_name() -> str: + return 'type' + + +class Object(BuiltIn): + @staticmethod + def get_name() -> str: + return 'object' + + +class File(BuiltIn): + @staticmethod + def get_name() -> str: + return 'file' + + +class AnyType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'Any' + + +# others: contextmanager, code (~str?), internal + +built_in_types = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltIn) and x is not BuiltIn and + x is not GenericType and x is not GenericBuiltInType) + + +def get_built_in_type(name: str) -> Optional[BuiltIn]: + bit = [x for x in built_in_types if x[1].get_name() == name] + assert len(bit) <= 1 + if len(bit) == 0: + return None + + return bit[0][1]() + + +def get_all_built_in_types() -> typing.List[BuiltIn]: + return [bit[1]() for bit in built_in_types] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_data.py b/plugins/python/parser/src/scripts/cc_python_parser/class_data.py new file mode 100644 index 000000000..39d3dda09 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_data.py @@ -0,0 +1,61 @@ +from typing import List, Optional + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.base_data import TypeDeclaration, ImportedDeclaration, DocumentedType +import cc_python_parser.function_data as fd +import cc_python_parser.variable_data as vd +from cc_python_parser.persistence.base_dto import UsageDTO + +from cc_python_parser.persistence.class_dto import ClassDeclarationDTO + + +# TODO: @classmethod +class ClassDeclaration(TypeDeclaration, DocumentedType): + def __init__(self, name: str, qualified_name: str, position: FilePosition, base_classes: List['ClassDeclaration'], + documentation: str): + TypeDeclaration.__init__(self, name, qualified_name, position) + DocumentedType.__init__(self, documentation) + self.base_classes: List[ClassDeclaration] = base_classes + self.methods: List[fd.FunctionDeclaration] = [] + self.static_methods: List[fd.StaticFunctionDeclaration] = [] + self.attributes: List[vd.VariableDeclaration] = [] + self.static_attributes: List[vd.StaticVariableDeclaration] = [] + self.classes: List[ClassDeclaration] = [] + + def create_dto(self) -> ClassDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + base_classes = set() + for base_class in self.base_classes: + base_classes.add(base_class.qualified_name) + members = ClassDeclarationDTO.ClassMembersDTO() + for m in self.methods: + members.methods.append(m.create_dto()) + for sm in self.static_methods: + members.static_methods.append(sm.create_dto()) + for a in self.attributes: + members.attributes.append(a.create_dto()) + for sa in self.static_attributes: + members.static_attributes.append(sa.create_dto()) + for c in self.classes: + members.classes.append(c.create_dto()) + return ClassDeclarationDTO(self.name, self.qualified_name, self.file_position, + types, usages, base_classes, members, self.documentation) + + def get_init_method(self) -> Optional[fd.FunctionDeclaration]: + for m in reversed(self.methods): + if m.name == '__init__': + return m + return None # has default init + + def get_type_repr(self) -> str: + return self.name + + +class ImportedClassDeclaration(ClassDeclaration, ImportedDeclaration[ClassDeclaration]): + def __init__(self, qualified_name: str, name: str, pos: FilePosition, class_declaration: ClassDeclaration, module): + ClassDeclaration.__init__(self, name, "", pos, class_declaration.base_classes, "") + ImportedDeclaration.__init__(self, qualified_name, class_declaration, module, pos) + self.type = {class_declaration} diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py b/plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py new file mode 100644 index 000000000..466451abb --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py @@ -0,0 +1,14 @@ +from typing import List, Union, Set, Optional + +import cc_python_parser.base_data as data +from cc_python_parser.built_in_types import BuiltIn +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.function_data import FunctionDeclaration + + +class ClassInitDeclaration(FunctionDeclaration): + def __init__(self, class_declaration: ClassDeclaration): + super().__init__(class_declaration.name, class_declaration.qualified_name, class_declaration.file_position, + [], "") + self.class_declaration = class_declaration diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py b/plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py new file mode 100644 index 000000000..fd260e558 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py @@ -0,0 +1,88 @@ +import ast +from pathlib import PurePath +from typing import List, Dict, Optional, Union + +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import has_attr, create_range_from_ast_node +from cc_python_parser.preprocessed_data import PreprocessedDeclaration +from cc_python_parser.preprocessed_function import PreprocessedFunction +from cc_python_parser.preprocessed_variable import PreprocessedVariable + + +class PreprocessedClass(PreprocessedDeclaration): + def __init__(self, name: str, file_position: FilePosition): + super().__init__(name, file_position) + self.methods: List[PreprocessedFunction] = [] + self.attributes: List[PreprocessedVariable] = [] + self.classes: List[PreprocessedClass] = [] + self.type_usages: List[Declaration] = [] + + def append_method(self, name: str, file_position: FilePosition, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]): + self.methods.append(PreprocessedFunction(name, file_position, node)) + + def append_attribute(self, name: str, file_position: FilePosition): + self.attributes.append(PreprocessedVariable(name, file_position)) + + def append_class(self, nested_class: 'PreprocessedClass'): # 'type' -> in case of type(self) + self.classes.append(nested_class) + + +class PreprocessedClassCollector(ast.NodeVisitor): + def __init__(self, path: Optional[PurePath]): + self.path = path + self.classes: List[PreprocessedClass] = [] + self.class_list: List[int] = [] + self.class_nest_class_map: Dict[PreprocessedClass, List[ast.ClassDef]] = {} + + def append_class(self, node: ast.ClassDef): + self.class_list.append(len(self.classes)) + preprocessed_class = PreprocessedClass(node.name, FilePosition(self.path, create_range_from_ast_node(node))) + self.classes.append(preprocessed_class) + self.handle_nested_class(node, preprocessed_class) + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.append_method(member, FilePosition(self.path, create_range_from_ast_node(member))) + elif isinstance(member, ast.Assign): + self.append_attribute(member, FilePosition(self.path, create_range_from_ast_node(member))) + elif isinstance(member, ast.ClassDef): + self.append_nested_class(preprocessed_class, member) + elif isinstance(member, ast.Pass): + pass + elif isinstance(member, ast.Expr) and hasattr(member, 'value') and isinstance(member.value, ast.Constant) \ + and hasattr(member.value, "value") and isinstance(member.value.value, str): + pass # documentation comment + else: + pass + # assert False, "Unknown class member: " + str(type(member)) + + def handle_nested_class(self, node: ast.ClassDef, preprocessed_class: PreprocessedClass): + for parent_class in self.class_nest_class_map: + if node in self.class_nest_class_map[parent_class]: + parent_class.append_class(preprocessed_class) + self.class_nest_class_map[parent_class].remove(node) + + def class_processed(self): + del self.class_list[-1] + + def append_method(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef], file_position: FilePosition): + self.classes[self.class_list[-1]].append_method(node.name, file_position, node) + if node.name == '__init__': + self.visit(node) + + def append_attribute(self, node: ast.Assign, file_position: FilePosition): + for attribute in node.targets: + if hasattr(attribute, 'id'): # class variable + self.classes[self.class_list[-1]].append_attribute(attribute.id, file_position) + elif hasattr(attribute, 'attr') and has_attr(attribute, 'value.id') and attribute.value.id == 'self': + self.classes[self.class_list[-1]].append_attribute(attribute.attr, file_position) + + def append_nested_class(self, node: PreprocessedClass, member: ast.ClassDef): + if node in self.class_nest_class_map: + self.class_nest_class_map[node].append(member) + else: + self.class_nest_class_map[node] = [member] + + def visit_Assign(self, node: ast.Assign): + self.append_attribute(node, FilePosition(self.path, create_range_from_ast_node(node))) + self.generic_visit(node) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py new file mode 100644 index 000000000..1f44934ab --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py @@ -0,0 +1 @@ +__all__ = ["hashable_list", "file_position", "parser_tree", "position", "utils"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py b/plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py new file mode 100644 index 000000000..de1bd0519 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py @@ -0,0 +1,29 @@ +from pathlib import PurePath +from typing import Optional + +from cc_python_parser.common.position import Range + + +class FilePosition: + def __init__(self, file: Optional[PurePath], r: Range): + self.file: Optional[PurePath] = file # Optional -> builtins + self.range: Range = r + + def __str__(self): + return "File: " + str(self.file) + " - " + str(self.range) + + def __repr__(self): + return self.__str__() + + def __hash__(self): + return hash(str(self.file)) ^ hash(self.range) + + def __eq__(self, other): + return self.file == other.file and self.range == other.range + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def get_empty_file_position(): + return FilePosition(None, Range.get_empty_range()) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py b/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py new file mode 100644 index 000000000..48d0a8cc0 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py @@ -0,0 +1,25 @@ +from typing import List, TypeVar, Generic + +T = TypeVar('T') + + +class OrderedHashableList(Generic[T], List[T]): + def __hash__(self): + return hash(e for e in self) + + def __add__(self, other): + new_list = OrderedHashableList() + new_list.extend(self) + new_list.extend(other) + return new_list + + def __eq__(self, other): + if not isinstance(other, type(self)) or len(other) != len(self): + return False + for i in range(0, len(self)): + if self[i] != other[i]: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py b/plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py new file mode 100644 index 000000000..44e7be21e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py @@ -0,0 +1,100 @@ +import datetime +from time import time + + +class PythonParserMetrics: + def __init__(self, project: str): + self.project: str = project + self.file_count: int = 0 + self.line_count: int = 0 + self.parse_time: float = 0 + self.function_count: int = 0 + self.class_count: int = 0 + self.import_count: int = 0 + self.ast_count: int = 0 + # type metrics + + def add_file_count(self, file_count: int = 1): + self.file_count += file_count + + def add_line_count(self, line_count: int = 1): + self.line_count += line_count + + def start_parsing(self): + self.parse_time = time() + + def stop_parsing(self): + self.parse_time = time() - self.parse_time + + def add_function_count(self, function_count: int = 1): + self.function_count += function_count + + def add_class_count(self, class_count: int = 1): + self.class_count += class_count + + def add_import_count(self, import_count: int = 1): + self.import_count += import_count + + def add_ast_count(self, ast_count: int = 1): + self.ast_count += ast_count + + def init(self, project: str): + self.project = project + self.file_count = 0 + self.line_count = 0 + self.parse_time = 0 + self.function_count = 0 + self.class_count = 0 + self.import_count = 0 + self.ast_count = 0 + # type metrics + + def __add__(self, other): + if not isinstance(other, PythonParserMetrics): + return self + new = self.__copy__() + new.file_count += other.file_count + new.line_count += other.line_count + new.parse_time += other.parse_time + new.function_count += other.function_count + new.class_count += other.class_count + new.import_count += other.import_count + new.ast_count += other.ast_count + return new + + def __iadd__(self, other): + return self.__add__(other) + + def __copy__(self): + copy = PythonParserMetrics(self.project) + copy.file_count = self.file_count + copy.line_count = self.line_count + copy.parse_time = self.parse_time + copy.function_count = self.function_count + copy.class_count = self.class_count + copy.import_count = self.import_count + copy.ast_count = self.ast_count + return copy + + def __deepcopy__(self, memodict={}): + return self.__copy__() + + def __str__(self): + return f""" + Metrics: + ============= + Project: {self.project} + File count: {self.file_count} + Line count: {self.line_count} + Parse time: {datetime.timedelta(seconds = self.parse_time)} + Function count: {self.function_count} + Class count: {self.class_count} + Import count: {self.import_count} + Ast count: {self.ast_count} + """ + + def __repr__(self): + return self.__str__() + + +metrics: PythonParserMetrics = PythonParserMetrics('') diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py b/plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py new file mode 100644 index 000000000..cc0d60dab --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py @@ -0,0 +1,30 @@ +import ast + + +class ParserTreeNode: + def __init__(self, node, parent: 'ParserTreeNode'): + self.node = node + self.parent = parent + self.children = [] + self.process_children() + + def process_children(self): + for child in ast.iter_child_nodes(self.node): + self.children.append(ParserTreeNode(child, self)) + + +class ParserTree: + def __init__(self, node): + self.root = ParserTreeNode(node, None) + + def find_node(self, node) -> ParserTreeNode: + return self.find_node_in_parent(self.root, node) + + def find_node_in_parent(self, parent: ParserTreeNode, node) -> ParserTreeNode: + if parent.node is node: + return parent + for child in parent.children: + n = self.find_node_in_parent(child, node) + if n is not None: + return n + return None diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/position.py b/plugins/python/parser/src/scripts/cc_python_parser/common/position.py new file mode 100644 index 000000000..f3fd389d7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/position.py @@ -0,0 +1,48 @@ +class Position: + def __init__(self, line: int, col: int): + self.line = line + self.column = col + + def __str__(self): + return "line: " + str(self.line) + ", column: " + str(self.column) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.line == other.line and self.column == other.column + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.line) ^ hash(self.column) + + @staticmethod + def get_empty_position(): + return Position(0, 0) + + +class Range: + def __init__(self, start_pos: Position, end_pos: Position): + self.start_position = start_pos + self.end_position = end_pos + + def __str__(self): + return "Start position: " + str(self.start_position) + " - End position: " + str(self.end_position) + + def __repr__(self): + return self.__str__() + + def __hash__(self): + return hash(self.start_position) ^ hash(self.end_position) + + def __eq__(self, other): + return self.start_position == other.start_position and self.end_position == other.end_position + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def get_empty_range(): + return Range(Position.get_empty_position(), Position.get_empty_position()) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py new file mode 100644 index 000000000..9a8a4409d --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py @@ -0,0 +1,43 @@ +import ast +from pathlib import PurePath, Path +from typing import Final, List, Callable + +from cc_python_parser.common.position import Range, Position +from cc_python_parser.persistence.persistence import ModelPersistence + + +ENCODINGS: Final[List[str]] = ['utf-8', 'iso-8859-1'] + + +def process_file_content(path: PurePath, func: Callable[[str, int], None], persistence: ModelPersistence) -> None: + if not Path(path).exists(): + return + for encoding in ENCODINGS: + try: + with open(str(path), "r", encoding=encoding) as source: + content = source.read() + source.seek(0) + func(content, len(source.readlines())) + except UnicodeDecodeError: + continue + except FileNotFoundError: + persistence.log_error(f"File not found: {path}") + continue + else: + return + persistence.log_error(f"Unhandled encoding in {str(path)}") + + +def has_attr(obj, attrs) -> bool: + for attr in attrs.split("."): + if hasattr(obj, attr): + obj = getattr(obj, attr) + else: + return False + return True + + +def create_range_from_ast_node(node: ast.AST) -> Range: + start_pos = Position(node.lineno, node.col_offset) + end_pos = Position(node.end_lineno, node.end_col_offset) + return Range(start_pos, end_pos) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py new file mode 100644 index 000000000..c95f0f693 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py @@ -0,0 +1,72 @@ +from enum import Enum, unique, auto +from pathlib import PurePath, Path +from typing import Optional + +from cc_python_parser.common.utils import process_file_content +from cc_python_parser.persistence.file_content_dto import FileContentDTO +from cc_python_parser.persistence.file_dto import FileDTO +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.symbol_collector import SymbolCollector +from cc_python_parser.persistence.persistence import ModelPersistence + + +@unique +class ProcessStatus(Enum): + WAITING = auto() + PREPROCESSED = auto() + PROCESSED = auto() + + +# TODO: expr_1; import; expr_2; - correct script -> not a problem +class FileInfo: + def __init__(self, path: PurePath, persistence: ModelPersistence): + self.path: PurePath = path + self.symbol_collector: Optional[SymbolCollector] = None + self.preprocessed_file: PreprocessedFile = PreprocessedFile(path) + self.status: ProcessStatus = ProcessStatus.WAITING + self.persistence: ModelPersistence = persistence + + def get_file(self): + return self.path.name + + def get_file_name(self): + return self.path.stem + + def preprocess_file(self, tree) -> None: + self.preprocessed_file.visit(tree) + self.status = ProcessStatus.PREPROCESSED + + def set_variable_collector(self, variable_collector: SymbolCollector) -> None: + self.symbol_collector = variable_collector + self.status = ProcessStatus.PROCESSED + + def create_dto(self) -> FileDTO: + file_dto = FileDTO() + file_dto.path = str(self.path) + file_dto.file_name = self.path.name + file_dto.timestamp = Path(self.path).stat().st_mtime + file_dto.content = FileContentDTO(self.get_content(self.path)) + file_dto.parent = str(self.path.parent) + file_dto.parse_status = self.get_parse_status(self.status) + file_dto.documentation = self.preprocessed_file.documentation + return file_dto + + def get_content(self, file: PurePath) -> str: + content = "" + + def handle_file_content(c, _): + nonlocal content + content = c + + process_file_content(file, handle_file_content, self.persistence) + return content + + @staticmethod + def get_parse_status(status: ProcessStatus) -> int: + if status == ProcessStatus.WAITING: + return 0 + elif status == ProcessStatus.PREPROCESSED: + return 1 + elif status == ProcessStatus.PROCESSED: + return 2 + assert False diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_data.py b/plugins/python/parser/src/scripts/cc_python_parser/function_data.py new file mode 100644 index 000000000..f8fdde439 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_data.py @@ -0,0 +1,52 @@ +from typing import List, Optional, Set, Union + +from cc_python_parser.built_in_types import AnyType, Method, Function, Type +from cc_python_parser.common.file_position import FilePosition +import cc_python_parser.base_data as data +from cc_python_parser.persistence.base_dto import UsageDTO +from cc_python_parser.persistence.function_dto import FunctionDeclarationDTO +from cc_python_parser.type_data import DeclarationType, PlaceholderType + + +class FunctionDeclaration(data.Declaration, data.DocumentedType): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, params: List[data.Declaration], + documentation: str, func_type: Optional[Set[Union[DeclarationType]]] = None): + data.Declaration.__init__(self, name, qualified_name, pos, func_type) + data.DocumentedType.__init__(self, documentation) + self.parameters: List[data.Declaration] = params + self.local_variables: List[data.Declaration] = [] + + def create_dto(self) -> FunctionDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + for t in self.type: + if isinstance(t, PlaceholderType): + types.add(AnyType().qualified_name) + # elif isinstance(t, MethodVariableDeclaration): + # types.add(Method().qualified_name) + # elif isinstance(t, FunctionVariableDeclaration): + # types.add(Function().qualified_name) + # elif isinstance(t, TypeVariableDeclaration): + # types.add(Type().qualified_name) + else: + types.add(t.qualified_name) + params = [] + for param in self.parameters: + params.append(param.qualified_name) + local_vars = [] + for local in self.local_variables: + local_vars.append(local.qualified_name) + return FunctionDeclarationDTO(self.name, self.qualified_name, self.file_position, + types, usages, self.documentation, params, local_vars) + + +class StaticFunctionDeclaration(FunctionDeclaration): + pass + + +class ImportedFunctionDeclaration(FunctionDeclaration, data.ImportedDeclaration[FunctionDeclaration]): + def __init__(self, qualified_name: str, name: str, pos: FilePosition, func_declaration: FunctionDeclaration, module): + FunctionDeclaration.__init__(self, name, "", pos, func_declaration.parameters, "", func_declaration.type) + data.ImportedDeclaration.__init__(self, qualified_name, func_declaration, module, pos) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py new file mode 100644 index 000000000..f58cb6b3a --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py @@ -0,0 +1,102 @@ +import ast +from typing import Union, Any, List, Optional, Set, Tuple + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.scope import FunctionScope +from cc_python_parser.symbol_collector import SymbolCollector +from cc_python_parser.symbol_collector_interface import IFunctionSymbolCollector +from cc_python_parser.type_data import DeclarationType +from cc_python_parser.type_deduction import TypeDeduction +from cc_python_parser.variable_data import VariableDeclaration + + +class TemporaryScope(FunctionScope): + pass + + +class FunctionSymbolCollector(SymbolCollector, IFunctionSymbolCollector): + def __init__(self, symbol_collector: SymbolCollector, root: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[Tuple[DeclarationType, Optional[str]]]): + SymbolCollector.__init__(self, root, symbol_collector.current_file, + symbol_collector.preprocessed_file, + symbol_collector.import_finder, + symbol_collector.scope_manager.persistence, + symbol_collector.function_symbol_collector_factory) + IFunctionSymbolCollector.__init__(self) + self.scope_manager = symbol_collector.scope_manager + self.type_deduction = TypeDeduction(self, self.scope_manager, self.preprocessed_file, + self.function_symbol_collector_factory, + symbol_collector.scope_manager.persistence) + self.imported_declaration_scope_map = symbol_collector.imported_declaration_scope_map + + self.arguments = arguments + self.current_function_calls = [] + if isinstance(symbol_collector, FunctionSymbolCollector): + self.current_function_calls.extend(symbol_collector.current_function_calls) + if len(symbol_collector.current_function_declaration) > 0: + self.current_function_calls.append(symbol_collector.current_function_declaration[-1]) + + def collect_symbols(self): + self.scope_manager.lock_persisting() + SymbolCollector.collect_symbols(self) + self.scope_manager.unlock_persisting() + + # TODO: handle local functions + # TODO: circular import + placeholder? + def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: + r = create_range_from_ast_node(node) + self.function = FunctionDeclaration(node.name, + self.scope_manager.get_qualified_name_from_current_scope( + node.name, r.start_position.line), + FilePosition(self.current_file, r), [], "") + + for call in reversed(self.current_function_calls): + if call.name == self.function.name and call.file_position == self.function.file_position: + return + # self.scope_manager.append_function_to_current_scope(self.function) + self.current_function_declaration.append(self.function) + + def action(): + self.collect_parameters(node) + self.generic_visit(node) + + self.scope_manager.with_scope(FunctionScope(node.name, + self.scope_manager.get_qualified_scope_name_from_current_scope(), + FilePosition.get_empty_file_position()), action) + del self.current_function_declaration[-1] + + def collect_parameters(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: + args_with_default = [] + for i in range(0, len(node.args.args)): + if i < len(node.args.defaults): + default = node.args.defaults[-(i+1)] + else: + default = None + args_with_default.append((node.args.args[-(i+1)], default)) + + arg_idx = 0 + for param in reversed(args_with_default): + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.\ + get_qualified_name_from_current_scope(param[0].arg, r.start_position.line) + new_variable = VariableDeclaration(param[0].arg, qualified_name, FilePosition(self.current_file, r)) + if param[0].arg == 'self' and self.scope_manager.is_current_scope_method() and \ + node.args.args[0] is param[0]: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + else: + # has kw -> has arg -> default + keyword = [k for k in self.arguments if k[1] == param[0].arg] + if len(keyword) == 0: + if arg_idx < len(self.arguments): + new_variable.type.update(self.type_deduction.get_current_type({self.arguments[arg_idx][0]})) + arg_idx += 1 + elif param[1] is not None: + new_variable.type.update(self.type_deduction.get_current_type( + self.type_deduction.deduct_type(param[1]))) + elif len(keyword) == 1: + new_variable.type.update(self.type_deduction.get_current_type({keyword[0][0]})) + else: + assert False + self.scope_manager.append_variable_to_current_scope(new_variable) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py new file mode 100644 index 000000000..4707a4cb9 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py @@ -0,0 +1,15 @@ +import ast +from abc import ABC, abstractmethod +from typing import Union, List, Optional, Tuple + +from cc_python_parser.symbol_collector_interface import SymbolCollectorBase, IFunctionSymbolCollector +from cc_python_parser.type_data import DeclarationType + + +class FunctionSymbolCollectorFactory(ABC): + @abstractmethod + def get_function_symbol_collector(self, symbol_collector: SymbolCollectorBase, + func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[Tuple[DeclarationType, Optional[str]]])\ + -> IFunctionSymbolCollector: + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/import_finder.py b/plugins/python/parser/src/scripts/cc_python_parser/import_finder.py new file mode 100644 index 000000000..93eb5238e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/import_finder.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from pathlib import PurePath +from typing import Optional + + +class ImportFinder(ABC): + # Optional[FileInfo] - circular import + @abstractmethod + def get_file_by_location(self, location: PurePath) -> Optional: + pass + + # Optional[GlobalScope] - circular import + def get_global_scope_by_location(self, location: PurePath) -> Optional: + file = self.get_file_by_location(location) + if file is not None and file.symbol_collector is not None: + assert file.symbol_collector.scope_manager.get_size() == 1 + return file.symbol_collector.scope_manager.get_global_scope() + elif file is not None: + return file.preprocessed_file + return None + + def get_scope_manager_by_location(self, location: PurePath) -> Optional: + file = self.get_file_by_location(location) + if file is not None and file.symbol_collector is not None: + assert file.symbol_collector.scope_manager.get_size() == 1 + return file.symbol_collector.scope_manager + return None diff --git a/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py new file mode 100644 index 000000000..6f05fef11 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py @@ -0,0 +1,178 @@ +import ast +from pathlib import PurePath +from typing import Optional, List, Set +from importlib import util # python 3.4 or above + +from cc_python_parser.common.position import Range +from cc_python_parser.common.utils import create_range_from_ast_node + + +class FileDependency: + def __init__(self, location: PurePath, module: str, alias: str = None): + self.location = location + self.module: str = module + self.alias: Optional[str] = alias + + def get_file(self) -> str: + module_parts = self.module.split('.') + return module_parts[-1] + '.py' + + +class ModuleImport: + def __init__(self, name: str, alias: Optional[str] = None): + self.name: str = name + self.alias: Optional[str] = alias + + +class SymbolImport: + def __init__(self, name: str, alias: Optional[str] = None): + self.name: str = name + self.alias: Optional[str] = alias + + def is_all_imported(self) -> bool: + return self.name == '*' + + +class Import: + def __init__(self, path: List[str], module: ModuleImport, imported: List[SymbolImport], + location: Optional[PurePath], r: Range, is_local: bool = False): + self.path: List[str] = path + self.module: ModuleImport = module + self.imported: List[SymbolImport] = imported + self.location: Optional[PurePath] = location + self.range = r + self.is_local = is_local + + def is_module_import(self): + return len(self.imported) == 0 + + +# TODO: need locations of files, and check if modules are among them? +# TODO: handle relative import: +# '.': from the package (parent directory) of the current module (from .'package' import 'module') +# from the __init__ (from . import 'symbol') +# '..': from the grandparent directory of the current module (and so on) +class ImportTable: + def __init__(self): + self.modules: List[Import] = [] + self.import_paths: Set[PurePath] = set() + + def append_import(self, node: ast.Import, is_local: bool = False): + self.modules.extend(self.convert_ast_import_to_import(node, is_local)) + for module in node.names: + m = None + try: + if module.name != '__main__': + m = util.find_spec(module.name) + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + pass + if m is not None and m.origin is not None: + self.import_paths.add(PurePath(m.origin)) + + def append_import_from(self, node: ast.ImportFrom, is_local: bool = False): + self.modules.extend(self.convert_ast_import_from_to_import(node, is_local)) + m = None + try: + if node.level > 0 and node.module is None: + m = util.find_spec(node.names[0].name) + elif node.level == 0 or node.module is not None: + m = util.find_spec(node.module + '.' + node.names[0].name) + else: + pass # print() + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + pass + except KeyError: + return + finally: + if m is None: + try: + if node.module != '__main__': + m = util.find_spec(node.module) + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + pass + except KeyError: + return + if m is not None and m.origin is not None: + self.import_paths.add(PurePath(m.origin)) + + @staticmethod + def convert_ast_import_to_import(node: ast.Import, is_local: bool = False) -> List[Import]: + imports = [] + for module in node.names: + module_parts = module.name.split('.') + module_name = module_parts[-1] + del module_parts[-1] + r = create_range_from_ast_node(node) + imports.append(ImportTable.create_import(module_parts, module_name, module.asname, [], r, is_local)) + return imports + + @staticmethod + def convert_ast_import_from_to_import(node: ast.ImportFrom, is_local: bool = False) -> List[Import]: + assert len(node.names) > 0 + is_module = True + try: + if node.level > 0 and node.module is None: + if util.find_spec(node.names[0].name) is None: + pass # print() + elif (node.level == 0 or node.module is not None) and \ + util.find_spec(node.module + '.' + node.names[0].name) is None: + is_module = False + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + # v3.7: before: AttributeError, after: ModuleNotFoundError + is_module = False + except KeyError: + return [] + imports = [] + r = create_range_from_ast_node(node) + if is_module: # modules + if node.module is None: # relative + module_path = [] + else: + module_path = node.module.split('.') + for module in node.names: + imports.append( + ImportTable.create_import(module_path, module.name, module.asname, [], r, is_local)) + else: # variables, functions + imported = [] + for module in node.names: + imported.append(SymbolImport(module.name, module.asname)) + module_parts = node.module.split('.') + module_name = module_parts[-1] + module_parts.remove(module_name) + imports.append(ImportTable.create_import(module_parts, module_name, None, imported, r, is_local)) + return imports + + @staticmethod + def create_import(path: List[str], name: str, alias: Optional[str], imported: List[SymbolImport], + r: Range, is_local: bool = False) -> Import: + location = None + module = path + module.append(name) + try: + if module[-1] == '__main__': # TODO: util.find_spec finds the module, but has no spec (exception) + spec = None + else: + spec = util.find_spec('.'.join(module)) + if spec is None: + pass # print() + # assert False, "Cannot find module: " + '.'.join(module) + elif spec.has_location: + location = spec.origin + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + # v3.7: before: AttributeError, after: ModuleNotFoundError + pass + optional_location = None + if location is not None: + optional_location = PurePath(location) + return Import(path, ModuleImport(name, alias), imported, optional_location, r, is_local) + + def get_dependencies(self) -> List[FileDependency]: + dependencies = [] + for module in self.modules: + if len(module.path) == 0: + dependencies.append(FileDependency(module.location, module.module.name, module.module.alias)) + else: + dependencies.append( + FileDependency( + module.location, '.'.join(module.path) + '.' + module.module.name, module.module.alias)) + return dependencies diff --git a/plugins/python/parser/src/scripts/cc_python_parser/logger.py b/plugins/python/parser/src/scripts/cc_python_parser/logger.py new file mode 100644 index 000000000..e1ae69a30 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/logger.py @@ -0,0 +1,16 @@ +import pathlib +import logging + +directory = pathlib.Path(__file__).resolve().parent + +logger = logging.getLogger('test') +logger.setLevel(logging.DEBUG) +file_handler = logging.FileHandler(str(directory) + '/test.log', 'w', encoding='utf-8') +file_handler.setLevel(logging.DEBUG) +logger.addHandler(file_handler) + +db_logger = logging.getLogger('db_test') +db_logger.setLevel(logging.DEBUG) +db_file_handler = logging.FileHandler(str(directory) + '/db_test.log', 'w', encoding='utf-8') +db_file_handler.setLevel(logging.DEBUG) +db_logger.addHandler(db_file_handler) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/main.py b/plugins/python/parser/src/scripts/cc_python_parser/main.py new file mode 100644 index 000000000..19c34d2ed --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/main.py @@ -0,0 +1,72 @@ +import os +import json +from copy import copy +from json import JSONDecodeError +from pathlib import PurePath, Path +from typing import List, Final + +from cc_python_parser.common.metrics import metrics, PythonParserMetrics +from cc_python_parser.parse_exception import ParseException +from cc_python_parser.parser import Parser +from cc_python_parser.persistence.persistence import ModelPersistence + + +def read_config() -> List[str]: + """ + Reads projects to parse from 'config.json', if it exists. Projects can be skipped with the "status" key: + { + "projects": [ + { + "project": "project_path", + "status": true/false + } + ] + } + """ + + config_file_name: Final[str] = 'config.json' + project_roots = [] + if Path(config_file_name).exists(): + with open(config_file_name, 'r') as config_file: + try: + config = json.load(config_file) + except JSONDecodeError as e: + print(f"Invalid JSON file: config.json (line: {e.lineno}, column: {e.colno})") + return project_roots + except Exception as e: + print(f"Error during config.json decoding: {e}") + return project_roots + for project in config['projects']: + if project['status']: + project_roots.append(project['project']) + return project_roots + +def main(): + def directory_exception(path: PurePath) -> bool: + directory = os.path.basename(os.path.normpath(str(path))) + return directory.startswith('.') or directory == 'venv' + + def file_exception(path: PurePath) -> bool: + return False + + project_roots = read_config() + + metrics_result = [] + for pr in project_roots: + metrics.init(pr) + exception = ParseException(directory_exception, file_exception) + p = Parser([pr], ModelPersistence(None), exception) + print(f"Parsing: {pr}") + p.parse() + metrics_result.append(copy(metrics)) + + all_metrics = PythonParserMetrics("ALL") + for m in metrics_result: + print(m) + all_metrics += m + + print(all_metrics) + + +if __name__ == '__main__': + main() diff --git a/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py new file mode 100644 index 000000000..17f601d58 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py @@ -0,0 +1,273 @@ +import ast +import sys +from typing import Optional, List, Any, Union + +from cc_python_parser.persistence.persistence import ModelPersistence + + +class MemberAccessCollector(ast.NodeVisitor): + class MemberData: + def __init__(self, name: Optional[str]): + self.name: str = name + + class ConstantData(MemberData): + pass + + class AttributeData(MemberData): + pass + + class MethodData(MemberData): + class Keyword: + def __init__(self, name: str, arg): + self.name: str = name + self.argument = arg + + def __init__(self, name: Optional[str], args, keywords): + super().__init__(name) + self.arguments = args + self.keywords = keywords + + class ReturnValueData(MemberData): + def __init__(self, name: Optional[str], member_data, node: ast.AST): + super().__init__(name) + self.return_of: MemberAccessCollector.MemberData = member_data + self.node = node + + class SubscriptData(MemberData): + class Index: + def __init__(self, idx): + self.idx = idx + + class Slice: + def __init__(self, lower, upper, step): + self.lower = lower + self.upper = upper + self.step = step + + def __init__(self, name: Optional[str], sub_slice: List[Union[Index, Slice]]): + super().__init__(name) + self.slice: List[Union[MemberAccessCollector.SubscriptData.Index, + MemberAccessCollector.SubscriptData.Slice]] = sub_slice + + class OperatorData(MemberData): + def __init__(self, name: Optional[str], node: Union[ast.BoolOp, ast.BinOp, ast.UnaryOp, ast.Compare]): + super().__init__(name) + + class LambdaData(MemberData): + def __init__(self, name: Optional[str], node: ast.Lambda): + super().__init__(name) + + class IfExpressionData(MemberData): + def __init__(self, name: Optional[str], node: ast.IfExp): + super().__init__(name) + + class ContainerLiteralData(MemberData): + def __init__(self, node): + super().__init__('') + self.container = node + + def __init__(self, member: Union[ast.Call, ast.Attribute], persistence: ModelPersistence): + self.call_list: List[MemberAccessCollector.MemberData] = [] + self.arguments = [] + self.last = False + self.persistence: ModelPersistence = persistence + self.visit(member) + + def generic_visit(self, node: ast.AST): + if self.last: + return + # await and starred must be skipped, and process the callable + if not isinstance(node, (ast.Await, ast.Starred)): + self.persistence.log_warning(f'Unhandled node type: {str(type(node))}') + ast.NodeVisitor.generic_visit(self, node) + + def visit_Call(self, node: ast.Call) -> Any: + if self.last: + return + + def process_call(value: ast.AST): + if isinstance(value, (ast.Name, ast.Attribute, ast.Lambda, ast.IfExp)): + self.call_list.append(self.MethodData(None, self.collect_arguments(node), self.collect_keywords(node))) + elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): + self.call_list.append( + self.ReturnValueData(None, self.MethodData(None, self.collect_arguments(node), + self.collect_keywords(node)), value)) + elif isinstance(value, ast.Await): + process_call(value.value) + elif isinstance(value, ast.IfExp): + pass # TODO + else: + assert False + process_call(node.func) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Subscript(self, node: ast.Subscript) -> Any: + if self.last: + return + + def process_subscript(value: ast.AST): + if isinstance(value, (ast.Name, ast.Attribute)): + self.call_list.append(self.SubscriptData(None, self.collect_slice(node))) + elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): + self.call_list.append( + self.ReturnValueData(None, self.SubscriptData(None, self.collect_slice(node)), value)) + elif isinstance(value, ast.Await): + process_subscript(value.value) + elif isinstance(value, (ast.Tuple, ast.List, ast.Dict, ast.Set)): + self.call_list.append(self.ContainerLiteralData(value)) + else: + assert False, "Not subscriptable type: " + str(type(value)) + process_subscript(node.value) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Attribute(self, node: ast.Attribute) -> Any: + if self.last: + return + if len(self.call_list) > 0 and self.call_list[-1].name is None: + self.call_list[-1].name = node.attr + else: + self.call_list.append(self.AttributeData(node.attr)) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Name(self, node: ast.Name) -> Any: + if self.last: + return + if len(self.call_list) == 0 or self.call_list[-1].name is not None: + self.call_list.append(self.AttributeData(node.id)) + else: + self.call_list[-1].name = node.id + self.last = True + + def visit_Constant(self, node: ast.Constant) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = node.value + else: + value = node.value + if isinstance(value, str): + value = '"' + value + '"' + self.call_list.append(self.ConstantData(value)) + self.last = True + + def visit_Lambda(self, node: ast.Lambda) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = 'lambda' + else: + self.call_list.append(self.LambdaData('lambda', node)) + self.last = True + + def visit_IfExp(self, node: ast.IfExp) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = 'ifexpression' + else: + self.call_list.append(self.IfExpressionData('IfExpression', node)) + self.last = True + + def visit_BinOp(self, node: ast.BinOp) -> Any: + self.last = True + + def visit_UnaryOp(self, node: ast.UnaryOp) -> Any: + self.last = True + + def visit_BoolOp(self, node: ast.BoolOp) -> Any: + self.last = True + + def visit_Compare(self, node: ast.Compare) -> Any: + self.last = True + + def visit_List(self, node: ast.List) -> Any: + self.last = True + + def visit_Tuple(self, node: ast.Tuple) -> Any: + self.last = True + + def visit_Dict(self, node: ast.Dict) -> Any: + self.last = True + + def visit_Set(self, node: ast.Set) -> Any: + self.last = True + + # TODO: is it necessary? + @staticmethod + def collect_arguments(call: ast.Call) -> List: + arguments = [] + for arg in call.args: + arguments.append(arg) + return arguments + + @staticmethod + def collect_keywords(call: ast.Call) -> List: + keywords = [] + for kw in call.keywords: + keywords.append(MemberAccessCollector.MethodData.Keyword(kw.arg, kw.value)) + return keywords + + @staticmethod + def collect_slice(subscript: ast.Subscript) -> List[Union[SubscriptData.Index, SubscriptData.Slice]]: + sub_slice = [] + + # since python 3.9 ast.Index, ast.ExtSlice are deprecated + if sys.version_info[0] == 3 and sys.version_info[1] >= 9: + def process_slice(node: (ast.Name, ast.Slice)): + if isinstance(node, ast.Slice): + sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) + elif isinstance(node, ast.Tuple): + for elem in node.elts: + process_slice(elem) + else: + sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) + else: + def process_slice(node: (ast.Index, ast.Slice, ast.ExtSlice)): + if isinstance(node, ast.Index): + sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) + elif isinstance(node, ast.Slice): + sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) + elif isinstance(node, ast.ExtSlice): + for dim in node.dims: + process_slice(dim) + else: + assert False, f"Unknown slice: str(type(node))" + + process_slice(subscript.slice) + return sub_slice + + @staticmethod + def is_callable_after_override(node: ast.AST) -> bool: + return isinstance(node, ast.BoolOp) or isinstance(node, ast.BinOp) or isinstance(node, ast.UnaryOp) or \ + isinstance(node, ast.Compare) + + +class MemberAccessCollectorIterator: + def __init__(self, member_access_collector: MemberAccessCollector): + self.mac = member_access_collector + self.index = len(self.mac.call_list) + + def __iter__(self): + self.index = len(self.mac.call_list) + return self + + def __next__(self): + if self.index <= 0: + raise StopIteration + self.index -= 1 + return self.mac.call_list[self.index] + + def get_current(self): + return self.mac.call_list[self.index] + + def get_first(self): + return self.mac.call_list[-1] + + def get_last(self): + return self.mac.call_list[0] + + def is_iteration_over(self) -> bool: + return self.index <= 0 + + def is_iteration_started(self) -> bool: + return self.index < len(self.mac.call_list) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py b/plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py new file mode 100644 index 000000000..2ff8a6448 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py @@ -0,0 +1,20 @@ +from pathlib import PurePath +from typing import Callable # python 3.5 or above + + +class ParseException: + def __init__(self, is_dictionary_exception: Callable[[PurePath], bool], + is_file_exception: Callable[[PurePath], bool]): + self.is_dictionary_exception = is_dictionary_exception + self.is_file_exception = is_file_exception + + def is_dictionary_exception(self, dictionary: PurePath) -> bool: + return self.is_dictionary_exception(dictionary) + + def is_file_exception(self, file: PurePath) -> bool: + return self.is_file_exception(file) + + +class DefaultParseException(ParseException): + def __init__(self): + super().__init__(lambda x: False, lambda x: False) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parser.py b/plugins/python/parser/src/scripts/cc_python_parser/parser.py new file mode 100644 index 000000000..d79e66b73 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/parser.py @@ -0,0 +1,151 @@ +import ast +import os +import sys +from pathlib import PurePath, Path +from typing import List, Optional, Union, Set, Tuple + +from cc_python_parser.common.utils import process_file_content +from cc_python_parser.file_info import FileInfo, ProcessStatus +from cc_python_parser.function_symbol_collector import FunctionSymbolCollector +from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.logger import logger, db_logger +from cc_python_parser.parse_exception import DefaultParseException, ParseException +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.scope import GlobalScope +from cc_python_parser.symbol_collector import SymbolCollector +from cc_python_parser.built_in_types import get_all_built_in_types +from cc_python_parser.built_in_functions import get_all_built_in_functions +from cc_python_parser.common.metrics import metrics + +# TODO: builtins - dir(builtins) +# TODO: change logging to DB storing +from cc_python_parser.symbol_collector_interface import IFunctionSymbolCollector +from cc_python_parser.type_data import DeclarationType + +current_file = "" # TODO: only for debug, remove it + + +class Parser(ast.NodeVisitor, ImportFinder, FunctionSymbolCollectorFactory): + def __init__(self, directories: List[str], persistence: ModelPersistence, + exceptions: Optional[ParseException] = None): + self.directories: List[str] = directories + self.other_modules: Set[PurePath] = set() + self.persistence: ModelPersistence = persistence + if exceptions is None: + exceptions = DefaultParseException() + self.exceptions: ParseException = exceptions + self.files: List[FileInfo] = [] + self.scope_managers = [] + self.collect_files() + + def collect_files(self): + for directory in self.directories: + sys.path.append(directory) + self.process_directory(PurePath(directory)) + + def process_directory(self, directory: PurePath): + for root, dirs, files in os.walk(directory, topdown=True, followlinks=True): + dirs[:] = [d for d in dirs if not self.exceptions.is_dictionary_exception(d)] + for file in files: + if self.exceptions.is_file_exception(file): + continue + file_path = Path(root, file) + if file_path.is_symlink(): + if not file_path.exists(): + continue + file_path = file_path.resolve() + if file_path.suffix == '.py': + self.files.append(FileInfo(file_path, self.persistence)) + + def parse(self) -> None: + metrics.start_parsing() + self.persist_builtins() + for file_info in self.files: + if file_info.symbol_collector is None: + self.parse_file(file_info) + self.persist_global_scopes() + metrics.stop_parsing() + + def parse_file(self, file_info: FileInfo) -> None: + global current_file + current_file = file_info.get_file() + if file_info.status != ProcessStatus.WAITING or current_file[-3:] != '.py': + return + logger.debug('\nFILE: ' + file_info.get_file_name() + '\n=========================================') + db_logger.debug('\nFILE: ' + file_info.get_file_name() + '\n=========================================') + if file_info.path is None: + return + + root = None + + def handle_file_content(c, line_num): + nonlocal root + try: + root = ast.parse(c) + metrics.add_line_count(line_num) + metrics.add_file_count() + except SyntaxError as e: + self.persistence.log_error(f"Syntax error in file {e.filename} at (line - {e.lineno}, column - {e.offset}): {e.text}") + + process_file_content(file_info.path, handle_file_content, self.persistence) + + if root is None: + return + file_info.preprocess_file(root) + self.handle_modules_outside_of_project(file_info) + for dependency in file_info.preprocessed_file.import_table.get_dependencies(): + dependency_file_info = [x for x in self.files if x.path == dependency.location] + if len(dependency_file_info) == 0: + pass + elif len(dependency_file_info) == 1: + if dependency_file_info[0].status is ProcessStatus.WAITING: + self.parse_file(dependency_file_info[0]) + current_file = file_info.get_file() + logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") + db_logger. \ + debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") + else: + assert False, 'Multiple file occurrence: ' + dependency.get_file() + sc = SymbolCollector(root, file_info.path, file_info.preprocessed_file, + self, self.persistence, self) + # print(current_file) + sc.collect_symbols() + file_info.set_variable_collector(sc) + assert(isinstance(sc.scope_manager.get_current_scope(), GlobalScope)) + self.scope_managers.append(sc.scope_manager) + # file_info.symbol_collector.scope_manager.persist_current_scope() + + self.persistence.persist_file(file_info.create_dto()) + + def handle_modules_outside_of_project(self, file_info: FileInfo) -> None: + for m in file_info.preprocessed_file.import_table.import_paths: + # TODO: after python 3.9 use PurePath.is_relative_to instead of 'in parents' + if not any(PurePath(mm) in m.parents for mm in self.directories) and \ + m not in self.other_modules: + self.other_modules.add(m) + new_file_info = FileInfo(m, self.persistence) + self.parse_file(new_file_info) + + def get_file_by_location(self, location: PurePath) -> Optional[FileInfo]: + for file in self.files: + if file.path == location: + return file + return None + + def persist_builtins(self): + for bit in get_all_built_in_types(): + self.persistence.persist_preprocessed_class(bit.create_dto()) + + for bif in get_all_built_in_functions(): + self.persistence.persist_function(bif.create_dto()) + + def persist_global_scopes(self): + for scope_manager in self.scope_managers: + scope_manager.persist_current_scope() + + def get_function_symbol_collector(self, symbol_collector: SymbolCollector, + func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[Tuple[DeclarationType, Optional[str]]]) \ + -> IFunctionSymbolCollector: + return FunctionSymbolCollector(symbol_collector, func_def, arguments) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py new file mode 100644 index 000000000..a435651d9 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py @@ -0,0 +1,2 @@ +__all__ = ["build_action", "build_source_target", "file_dto", "file_content_dto", "persistence", "class_dto", + "function_dto", "import_dto", "variable_dto", "base_dto"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py new file mode 100644 index 000000000..62e26bde7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py @@ -0,0 +1,36 @@ +from typing import List, Set + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.file_position_dto import FilePositionDTO + +PUBLIC_DECLARATION = "public" +SEMI_PRIVATE_DECLARATION = "semiprivate" +PRIVATE_DECLARATION = "private" + + +def create_file_position_dto(file_position: FilePosition) -> FilePositionDTO: + return FilePositionDTO(str(file_position.file), file_position.range) + + +class UsageDTO: + def __init__(self, file_position: FilePosition): + self.file_position: FilePositionDTO = create_file_position_dto(file_position) + + +class DeclarationDTO: + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO]): + self.name: str = name + self.qualified_name: str = qualified_name + self.file_position: FilePositionDTO = create_file_position_dto(file_position) + self.type: List[str] = list(types) # qualified names + self.usages: List[UsageDTO] = usages + self.visibility: str = self.get_visibility() + + def get_visibility(self) -> str: + if self.name.startswith('__'): + return PRIVATE_DECLARATION + elif self.name.startswith('_'): + return SEMI_PRIVATE_DECLARATION + else: + return PUBLIC_DECLARATION diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py new file mode 100644 index 000000000..6a5b428b7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py @@ -0,0 +1,11 @@ +from typing import List + +import cc_python_parser.persistence.build_source_target as bst + + +class BuildAction: + def __init__(self, command: str, sources: List[bst.BuildSource], targets: List[bst.BuildTarget]): + self.command = command + self.type = 0 # compile + self.build_sources: List[bst.BuildSource] = sources + self.build_target: List[bst.BuildTarget] = targets diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py new file mode 100644 index 000000000..21accab05 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py @@ -0,0 +1,14 @@ +import cc_python_parser.persistence.build_action as ba +from cc_python_parser.persistence.file_dto import FileDTO + + +class BuildSource: + def __init__(self, file: FileDTO, action: ba.BuildAction): + self.file: FileDTO = file + self.action: ba.BuildAction = action + + +class BuildTarget: + def __init__(self, file: FileDTO, action: ba.BuildAction): + self.file: FileDTO = file + self.action: ba.BuildAction = action diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py new file mode 100644 index 000000000..ea8263929 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py @@ -0,0 +1,25 @@ +from typing import Set, List + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.base_dto import UsageDTO, DeclarationDTO +from cc_python_parser.persistence.documented_dto import DocumentedDTO +from cc_python_parser.persistence.variable_dto import VariableDeclarationDTO +from cc_python_parser.persistence.function_dto import FunctionDeclarationDTO + + +class ClassDeclarationDTO(DeclarationDTO, DocumentedDTO): + class ClassMembersDTO: + def __init__(self): + self.methods: List[FunctionDeclarationDTO] = [] + self.static_methods: List[FunctionDeclarationDTO] = [] + self.attributes: List[VariableDeclarationDTO] = [] + self.static_attributes: List[VariableDeclarationDTO] = [] + self.classes: List[ClassDeclarationDTO] = [] + + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, + types: Set[str], usages: List[UsageDTO], base_classes: Set[str], + members: 'ClassDeclarationDTO.ClassMembersDTO', documentation: str): + DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) + DocumentedDTO.__init__(self, documentation) + self.base_classes: List[str] = list(base_classes) + self.members: ClassDeclarationDTO.ClassMembersDTO = members diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py new file mode 100644 index 000000000..271e09b67 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py @@ -0,0 +1,3 @@ +class DocumentedDTO: + def __init__(self, documentation: str): + self.documentation: str = documentation diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py new file mode 100644 index 000000000..1487d9259 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py @@ -0,0 +1,3 @@ +class FileContentDTO: + def __init__(self, content: str): + self.content: str = content diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py new file mode 100644 index 000000000..459fc7853 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py @@ -0,0 +1,23 @@ +from enum import unique, Enum + +from cc_python_parser.persistence.documented_dto import DocumentedDTO +from cc_python_parser.persistence.file_content_dto import FileContentDTO + + +@unique +class ParseStatus(Enum): + PSNONE = 0 + PSPARTIALLYPARSED = 1 + PSFULLYPARSED = 2 + + +class FileDTO(DocumentedDTO): + def __init__(self): + super().__init__("") + self.type: str = 'Python' # dir, unknown, binary? + self.path: str = "" + self.file_name: str = "" + self.timestamp = "" + self.content: FileContentDTO = FileContentDTO("") + self.parent: str = "" + self.parse_status: int = 0 # PSNONE diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py new file mode 100644 index 000000000..6a374a823 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py @@ -0,0 +1,16 @@ +from cc_python_parser.common.position import Range + + +class FilePositionDTO: + def __init__(self, file: str, r: Range): + self.file: str = file + self.range: Range = r + + def __hash__(self): + return hash(str(self.file)) ^ hash(self.range) + + def __eq__(self, other): + return self.file == other.file and self.range == other.range + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py new file mode 100644 index 000000000..0e51ea271 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py @@ -0,0 +1,14 @@ +from typing import Set, List + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.base_dto import DeclarationDTO, UsageDTO +from cc_python_parser.persistence.documented_dto import DocumentedDTO + + +class FunctionDeclarationDTO(DeclarationDTO, DocumentedDTO): + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO], documentation: str, params: List[str], local_vars: List[str]): + DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) + DocumentedDTO.__init__(self, documentation) + self.parameters: List[str] = params + self.locals: List[str] = local_vars diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py new file mode 100644 index 000000000..181110eaa --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py @@ -0,0 +1,38 @@ +from pathlib import PurePath +from typing import Dict, List + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.base_dto import create_file_position_dto +from cc_python_parser.persistence.file_position_dto import FilePositionDTO + + +class ImportDataDTO: + def __init__(self, qualified_name: str, imported: PurePath, pos: FilePosition): + self.qualified_name: str = qualified_name + self.imported: str = str(imported) + self.position: FilePositionDTO = create_file_position_dto(pos) + + def __hash__(self): + return hash(self.imported) ^ hash(self.position) + + def __eq__(self, other): + return self.imported == other.imported and self.position == other.position + + def __ne__(self, other): + return not self.__eq__(other) + + +class ImportDTO: + def __init__(self, importer: PurePath): + self.importer: str = str(importer) + self.imported_modules: List[ImportDataDTO] = [] + self.imported_symbols: Dict[ImportDataDTO, List[str]] = {} + + def add_module_import(self, qualified_name: str, imported: PurePath, pos: FilePosition): + self.imported_modules.append(ImportDataDTO(qualified_name, imported, pos)) + + def add_symbol_import(self, qualified_name: str, imported: PurePath, pos: FilePosition, symbol_qualified_name: str): + imported_data_dto = ImportDataDTO(qualified_name, imported, pos) + if imported_data_dto not in self.imported_symbols: + self.imported_symbols[imported_data_dto] = [] + self.imported_symbols[imported_data_dto].append(symbol_qualified_name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py new file mode 100644 index 000000000..dab5eab44 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py @@ -0,0 +1,112 @@ +from pathlib import PurePath + +from cc_python_parser.persistence.file_dto import FileDTO +from cc_python_parser.persistence.variable_dto import VariableDeclarationDTO +from cc_python_parser.persistence.function_dto import FunctionDeclarationDTO +from cc_python_parser.persistence.class_dto import ClassDeclarationDTO +from cc_python_parser.persistence.import_dto import ImportDTO + + +class ModelPersistence: + def __init__(self, c_persistence): + self.c_persistence = c_persistence + self.check_c_persistence() + self.log = open(str(PurePath(__file__).parent.joinpath('persistence_log.txt')), 'w+', encoding='utf-8') + + def __del__(self): + self.log.close() + + def check_c_persistence(self): + if self.c_persistence is None: + return + assert hasattr(self.c_persistence, 'log_info') + assert hasattr(self.c_persistence, 'log_warning') + assert hasattr(self.c_persistence, 'log_error') + assert hasattr(self.c_persistence, 'log_debug') + assert hasattr(self.c_persistence, 'persist_file') + assert hasattr(self.c_persistence, 'persist_variable') + assert hasattr(self.c_persistence, 'persist_function') + assert hasattr(self.c_persistence, 'persist_preprocessed_class') + assert hasattr(self.c_persistence, 'persist_class') + assert hasattr(self.c_persistence, 'persist_import') + + def log_info(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_info(message) + else: + self.log.write(message) + + def log_warning(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_warning(message) + else: + self.log.write(message) + + def log_error(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_error(message) + else: + self.log.write(message) + + def log_debug(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_debug(message) + else: + self.log.write(message) + + def persist_file(self, file: FileDTO) -> None: + if self.c_persistence is not None: + self.log_debug(f"Persist file: {file.path}") + self.c_persistence.persist_file(file) + else: + self.log.write(f"Persist file: {file.path}\n") + + def persist_variable(self, declaration: VariableDeclarationDTO) -> None: + if self.c_persistence is not None: + self.log_debug(f"Persist var: {declaration.qualified_name}") + self.c_persistence.persist_variable(declaration) + else: + self.log.write(f"Persist var: {declaration.qualified_name}\n") + + def persist_function(self, declaration: FunctionDeclarationDTO) -> None: + if self.c_persistence is not None: + self.log_debug(f"Persist func: {declaration.qualified_name}") + self.c_persistence.persist_function(declaration) + else: + self.log.write(f"Persist func: {declaration.qualified_name}\n") + + def persist_preprocessed_class(self, declaration: ClassDeclarationDTO) -> None: + if self.c_persistence is not None: + self.log_debug(f"Persist preprocessed class: {declaration.qualified_name}") + self.c_persistence.persist_preprocessed_class(declaration) + else: + self.log.write(f"Persist preprocessed class: {declaration.qualified_name}\n") + + def persist_class(self, declaration: ClassDeclarationDTO) -> None: + if self.c_persistence is not None: + self.log_debug(f"Persist class: {declaration.qualified_name}") + self.c_persistence.persist_class(declaration) + else: + self.log.write(f"Persist class: {declaration.qualified_name}\n") + + def persist_import(self, imports: ImportDTO) -> None: + if self.c_persistence is not None: + self.log_debug("Persist import") + for i in imports.imported_modules: + self.log_debug(f"Persist imported module: {i.qualified_name}") + for i in imports.imported_symbols: + self.log_debug(f"Persist imported symbol: {i.qualified_name}") + self.c_persistence.persist_import(imports) + else: + for i in imports.imported_modules: + self.log.write(f"Persist imported module: {i.qualified_name}\n") + for i in imports.imported_symbols: + self.log.write(f"Persist imported symbol: {i.qualified_name}\n") + + +model_persistence = ModelPersistence(None) + + +def init_persistence(c_persistence): + global model_persistence + model_persistence = ModelPersistence(c_persistence) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py new file mode 100644 index 000000000..fcbb67938 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py @@ -0,0 +1,5 @@ +from cc_python_parser.persistence.base_dto import DeclarationDTO + + +class VariableDeclarationDTO(DeclarationDTO): + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py b/plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py new file mode 100644 index 000000000..aca4b859d --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py @@ -0,0 +1,52 @@ +import ast +from typing import Dict, Tuple, Optional, Set, Union, List + +from cc_python_parser.common.hashable_list import OrderedHashableList +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.type_data import DeclarationType + + +# TODO: redefined functions? +class PlaceholderFunctionDeclarationCache: + class FunctionDeclarationData: + def __init__(self, func: Union[ast.FunctionDef, ast.AsyncFunctionDef], is_method: bool): + self.function = func + self.is_method = is_method + + def __init__(self): + self.cache: \ + Dict[FunctionDeclaration, + Tuple[PlaceholderFunctionDeclarationCache.FunctionDeclarationData, + Dict[OrderedHashableList[DeclarationType], Set[DeclarationType]]]] = {} + + def add_function_declaration(self, declaration: FunctionDeclaration, function_def: FunctionDeclarationData) -> None: + self.cache[declaration] = (function_def, {}) + + def get_function_def(self, declaration: FunctionDeclaration) -> Optional[FunctionDeclarationData]: + if declaration in self.cache: + return self.cache[declaration][0] + return None + + def get_function_return_type(self, declaration: FunctionDeclaration, + param_types: OrderedHashableList[DeclarationType]) -> Optional[Set[DeclarationType]]: + if declaration in self.cache and param_types in self.cache[declaration][1]: + return self.cache[declaration][1][param_types] + return None + + def get_functions_return_type(self, declarations: Set[FunctionDeclaration], + param_types: OrderedHashableList[DeclarationType]) \ + -> Set[DeclarationType]: + types = set() + for declaration in declarations: + t = self.get_function_return_type(declaration, param_types) + if t is not None: + types.update(t) + return types + + def add_function_return_type(self, declaration: FunctionDeclaration, + param_types: OrderedHashableList[DeclarationType], return_type: Set[DeclarationType]) \ + -> None: + if declaration in self.cache and param_types not in self.cache[declaration][1]: + self.cache[declaration][1][param_types] = return_type + elif declaration not in self.cache: + assert False, "Key not found" diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py new file mode 100644 index 000000000..d28bb7895 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py @@ -0,0 +1,7 @@ +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition + + +class PreprocessedDeclaration(Declaration): + def __init__(self, name: str, file_position: FilePosition): + Declaration.__init__(self, name, "", file_position) # TODO: need qualified name? diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py new file mode 100644 index 000000000..0bf192797 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py @@ -0,0 +1,73 @@ +import ast +from pathlib import PurePath +from typing import Set, Any, Optional + +from cc_python_parser.class_preprocessor import PreprocessedClassCollector +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.import_preprocessor import ImportTable +from cc_python_parser.preprocessed_function import PreprocessedFunction +from cc_python_parser.preprocessed_variable import PreprocessedVariable + + +class PreprocessedFile(ast.NodeVisitor): + def __init__(self, path: Optional[PurePath]): + self.path = path + self.class_collector = PreprocessedClassCollector(path) + self.preprocessed_functions: Set[PreprocessedFunction] = set() + self.preprocessed_variables: Set[PreprocessedVariable] = set() + self.import_table = ImportTable() + self.depth = 0 + self.documentation: str = "" + + def visit_Module(self, node: ast.Module) -> Any: + documentation = ast.get_docstring(node) + if documentation is not None: + self.documentation = documentation + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> Any: + is_local = False + if self.depth > 0: + is_local = True + self.import_table.append_import(node, is_local) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: + is_local = False + if self.depth > 0: + is_local = True + self.import_table.append_import_from(node, is_local) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> Any: + self.class_collector.append_class(node) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + self.class_collector.class_processed() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + if self.depth == 0: + self.preprocessed_functions.add( + PreprocessedFunction(node.name, FilePosition(self.path, create_range_from_ast_node(node)), node)) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: + if self.depth == 0: + self.preprocessed_functions.add( + PreprocessedFunction(node.name, FilePosition(self.path, create_range_from_ast_node(node)), node)) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_Assign(self, node: ast.Assign) -> Any: + if self.depth == 0: + for target in node.targets: + if isinstance(target, ast.Name): + self.preprocessed_variables.add( + PreprocessedVariable(target.id, FilePosition(self.path, create_range_from_ast_node(node)))) + else: + pass # print("...") diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py new file mode 100644 index 000000000..317ad0fca --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py @@ -0,0 +1,22 @@ +import ast +from typing import List, Union + +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.preprocessed_data import PreprocessedDeclaration + + +class PreprocessedFunction(PreprocessedDeclaration): + def __init__(self, name: str, file_position: FilePosition, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]): + super().__init__(name, file_position) + self.type_usages: List[Declaration] = [] + self.node: Union[ast.FunctionDef, ast.AsyncFunctionDef] = node + + def __eq__(self, other): + return isinstance(other, PreprocessedFunction) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py new file mode 100644 index 000000000..d120e32f6 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py @@ -0,0 +1,20 @@ +from typing import List + +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.preprocessed_data import PreprocessedDeclaration + + +class PreprocessedVariable(PreprocessedDeclaration): + def __init__(self, name: str, file_position: FilePosition): + super().__init__(name, file_position) + self.type_usages: List[Declaration] = [] + + def __eq__(self, other): + return isinstance(other, PreprocessedVariable) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py b/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py new file mode 100644 index 000000000..d2ed1e086 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py @@ -0,0 +1,13 @@ +from cc_python_parser.parser import Parser +from cc_python_parser.skip_file import read_skip_file +from cc_python_parser.persistence.persistence import init_persistence, ModelPersistence + + +def parse(source_root: str, persistence): + model_persistence = ModelPersistence(persistence) + p = Parser([source_root], model_persistence, read_skip_file(model_persistence)) + p.parse() + + +if __name__ == '__main__': + parse("dummy", None) # error! diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope.py b/plugins/python/parser/src/scripts/cc_python_parser/scope.py new file mode 100644 index 000000000..6c093fa3e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope.py @@ -0,0 +1,223 @@ +from typing import List, Optional +from abc import abstractmethod, ABC +from uuid import uuid4 + +from cc_python_parser.base_data import Declaration, Usage +from cc_python_parser.built_in_types import String, Dictionary +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.import_preprocessor import ImportTable +from cc_python_parser.logger import logger +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.type_data import InitVariablePlaceholderType +from cc_python_parser.variable_data import VariableDeclaration + + +def create_qualified_name(scope_qualified_name: str, current_scope: Optional[str], symbol_name: str, line: int): + qualified_name = scope_qualified_name + if current_scope is not None: + qualified_name = qualified_name + '.' + current_scope + qualified_name = qualified_name + '.' + symbol_name + ':' + str(line) + return qualified_name + + +class Scope(ABC): + def __init__(self, name: str): + self.name: str = name + self.variable_declarations: List[VariableDeclaration] = [] + self.function_declarations: List[FunctionDeclaration] = [] + self.class_declarations: List[ClassDeclaration] = [] + # TODO: should use only self.declarations, and filter the others? (slower, but less memory) + self.declarations: List[Declaration] = [] + + def append_variable(self, variable_declaration: VariableDeclaration) -> None: + self.variable_declarations.append(variable_declaration) + self.declarations.append(variable_declaration) + logger.debug('Var: ' + variable_declaration.name + ' (' + str(variable_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ' + variable_declaration.get_type_repr()) + + def append_function(self, function_declaration: FunctionDeclaration) -> None: + self.function_declarations.append(function_declaration) + self.declarations.append(function_declaration) + logger.debug('Func: ' + function_declaration.name + ' (' + str(function_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ' + function_declaration.get_type_repr()) + + def append_class(self, class_declaration: ClassDeclaration) -> None: + self.class_declarations.append(class_declaration) + self.declarations.append(class_declaration) + logger.debug('Class: ' + class_declaration.name + ' (' + str(class_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ') + + def get_variable(self, usage: Usage) -> Optional[VariableDeclaration]: + return self.get_variable_by_name(usage.name) + + def get_variable_by_name(self, name: str) -> Optional[VariableDeclaration]: + variables = [x for x in self.variable_declarations if x.name == name] + # TODO: fix this assert + # assert len(variables) <= 1 + if len(variables) > 0: + return variables[0] + else: + return None + + def get_function(self, usage: Usage) -> Optional[FunctionDeclaration]: + return self.get_function_by_name(usage.name) + + def get_function_by_name(self, name: str) -> Optional[FunctionDeclaration]: + functions = [x for x in self.function_declarations if x.name == name] + if len(functions) > 0: + return functions[-1] + else: + return None + + def get_class_by_name(self, name: str) -> Optional[ClassDeclaration]: + classes = [x for x in self.class_declarations if x.name == name] + if len(classes) > 0: + return classes[-1] + else: + return None + + @abstractmethod + def get_type(self) -> str: + pass + + +class LifetimeScope(Scope, ABC): + """Symbols declared inside this scope are deleted after exiting from it.""" + + def __init__(self, name: str): + super().__init__(name) + + +class ImportScope(ABC): + """Imported modules are available during this scope.""" + + def __init__(self): + self.import_table: ImportTable = ImportTable() + + +class GlobalScope(LifetimeScope, ImportScope): + """The first scope, which lives on with its module.""" + + def __init__(self, qualified_scope_name: str, file_position: FilePosition): + LifetimeScope.__init__(self, '') + ImportScope.__init__(self) + self.append_variable(VariableDeclaration('__dict__', + create_qualified_name(qualified_scope_name, None, '__dict__', + file_position.range.start_position.line), + file_position, {Dictionary()})) + self.append_variable(VariableDeclaration('__doc__', + create_qualified_name(qualified_scope_name, None, '__doc__', + file_position.range.start_position.line), + file_position, {String()})) + self.append_variable(VariableDeclaration('__module__', + create_qualified_name(qualified_scope_name, None, '__module__', + file_position.range.start_position.line), + file_position, {String()})) + + def get_type(self) -> str: + return "global" + + +class FunctionScope(LifetimeScope, ImportScope): + """Scope of the functions and methods.""" + + def __init__(self, name: str, qualified_scope_name: str, file_position: FilePosition): + LifetimeScope.__init__(self, name) + ImportScope.__init__(self) + self.append_variable(VariableDeclaration('__dict__', + create_qualified_name(qualified_scope_name, name, '__dict__', + file_position.range.start_position.line), + file_position, {Dictionary()})) + self.append_variable(VariableDeclaration('__doc__', + create_qualified_name(qualified_scope_name, name, '__doc__', + file_position.range.start_position.line), + file_position, {String()})) + self.append_variable(VariableDeclaration('__module__', + create_qualified_name(qualified_scope_name, name, '__module__', + file_position.range.start_position.line), + file_position, {String()})) + + def get_type(self) -> str: + return "function" + + +class ClassScope(LifetimeScope, ImportScope): + """Scope of the classes.""" + + def __init__(self, name: str, qualified_scope_name: str, file_position: FilePosition): + LifetimeScope.__init__(self, name) + ImportScope.__init__(self) + self.init_placeholders: List[InitVariablePlaceholderType] = [] + self.append_variable(VariableDeclaration('__dict__', + create_qualified_name(qualified_scope_name, name, '__dict__', + file_position.range.start_position.line), + file_position, {Dictionary()})) + self.append_variable(VariableDeclaration('__doc__', + create_qualified_name(qualified_scope_name, name, '__doc__', + file_position.range.start_position.line), + file_position, {String()})) + self.append_variable(VariableDeclaration('__module__', + create_qualified_name(qualified_scope_name, name, '__module__', + file_position.range.start_position.line), + file_position, {String()})) + + def get_type(self) -> str: + return "class" + + +class PartialLifetimeScope(LifetimeScope, ABC): + """Not all of the symbols are deleted after existing from it.""" + pass + + +class ExceptionScope(PartialLifetimeScope): + """The exception variable is deleted after the 'except' scope, + but all the other symbols declared in it, are alive after it.""" + + def __init__(self): + super().__init__('exception') + + def get_type(self) -> str: + return 'exception' + + +class LambdaScope(LifetimeScope): + """Scope of lambdas.""" + + def __init__(self): + super().__init__('lambda_' + uuid4().hex) + + def get_type(self) -> str: + return "lambda" + + +class GeneratorScope(LifetimeScope): + """Scope of generators.""" + + def __init__(self): + super().__init__('generator_' + uuid4().hex) + + def get_type(self) -> str: + return "generator" + + +class ConditionalScope(Scope): + """Scope of if.""" + + def __init__(self): + super().__init__('if') + + def get_type(self) -> str: + return "conditional" + + +# TODO: need WhileScope and ForScope? +class LoopScope(Scope): + """Scope of for and while.""" + + def __init__(self): + super().__init__('loop') + + def get_type(self) -> str: + return "loop" diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py new file mode 100644 index 000000000..c5cfd7cc4 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py @@ -0,0 +1,504 @@ +import ast +from pathlib import PurePath, Path +from typing import List, Type, Iterator, Callable, Optional, Union + +from cc_python_parser.base_data import Declaration, ImportedDeclaration +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.class_init_declaration import ClassInitDeclaration +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.import_preprocessor import ImportTable, Import +from cc_python_parser.member_access_collector import MemberAccessCollectorIterator, MemberAccessCollector +from cc_python_parser.persistence.import_dto import ImportDTO +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.placeholder_function_declaration_cache import PlaceholderFunctionDeclarationCache +from cc_python_parser.scope import Scope, ClassScope, GlobalScope, FunctionScope, LifetimeScope, PartialLifetimeScope, \ + ConditionalScope, LoopScope, ImportScope +from cc_python_parser.type_data import PlaceholderType +from cc_python_parser.variable_data import VariableDeclaration, ModuleVariableDeclaration, FunctionVariableDeclaration, \ + TypeVariableDeclaration +from cc_python_parser.common.position import Range +from cc_python_parser.common.file_position import FilePosition + + +class ScopeManager: + def __init__(self, current_file: PurePath, import_finder: ImportFinder, persistence: ModelPersistence, + scopes: Optional[List[Scope]] = None): + self.current_file = current_file + self.import_finder = import_finder + if scopes is None: + self.scopes: List[Scope] = [GlobalScope(self.get_qualified_scope_name_from_current_scope([]), + FilePosition(current_file, Range.get_empty_range()))] + else: + self.scopes = scopes + self.persistence: ModelPersistence = persistence + self.placeholder_function_declaration_cache = PlaceholderFunctionDeclarationCache() + self.persist_symbols: int = 0 + + def lock_persisting(self): + self.persist_symbols += 1 + + def unlock_persisting(self): + self.persist_symbols -= 1 + + def is_current_scope_instance(self, t: Type) -> bool: + return isinstance(self.scopes[-1], t) + + # TODO: local functions + def is_current_scope_method(self) -> bool: + for i in range(len(self.scopes) - 1, -1, -1): + if isinstance(self.scopes[i], LifetimeScope) and not isinstance(self.scopes[i], PartialLifetimeScope): + return isinstance(self.scopes[i], FunctionScope) and isinstance(self.scopes[i - 1], ClassScope) + return False + + def is_current_scope_init(self) -> bool: + return self.is_current_scope_method() and self.get_current_lifetime_scope().name == '__init__' + + def get_current_lifetime_scope(self) -> LifetimeScope: + for scope in self.reverse(): + if isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): + return scope + assert False # there should be always a GlobalScope, which is LifetimeScope + + def reverse(self) -> Iterator[Scope]: + for scope in reversed(self.scopes): + yield scope + + def iterate(self) -> Iterator[Scope]: + for scope in self.scopes: + yield scope + + def with_scope(self, scope: Scope, action: Callable[[], None]) -> None: + self.scopes.append(scope) + action() + self.persist_current_scope() + del self.scopes[-1] + + def get_size(self) -> int: + return len(self.scopes) + + def get_current_scope(self) -> Scope: + return self.scopes[-1] + + def get_global_scope(self) -> GlobalScope: + assert len(self.scopes) > 0 + global_scope = self.scopes[0] + if not isinstance(global_scope, GlobalScope): + assert False + return global_scope + + def append_variable_to_current_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_lifetime_scope().append_variable(new_variable) + + def append_exception_to_current_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_partial_lifetime_scope().append_variable(new_variable) + + def append_function_to_current_scope(self, new_function: FunctionDeclaration) -> None: + self.get_current_lifetime_scope().append_function(new_function) + + def append_class_to_current_scope(self, new_class: ClassDeclaration) -> None: + self.get_current_lifetime_scope().append_class(new_class) + + def append_variable_to_current_class_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_class_scope().append_variable(new_variable) + + def append_function_to_current_class_scope(self, new_function: FunctionDeclaration) -> None: + self.get_current_class_scope().append_function(new_function) + + def append_class_to_current_class_scope(self, new_class: ClassDeclaration) -> None: + self.get_current_class_scope().append_class(new_class) + + def append_module_variable_to_current_scope(self, module_variable: ModuleVariableDeclaration) -> None: + def is_same_declaration(var: VariableDeclaration) -> bool: + return isinstance(var, ModuleVariableDeclaration) and \ + var.imported_module_location == module_variable.imported_module_location + + scope = self.get_current_lifetime_scope() + if len([v for v in scope.variable_declarations if is_same_declaration(v)]) == 0: + scope.append_variable(module_variable) + + def get_current_partial_lifetime_scope(self) -> Optional[LifetimeScope]: + for scope in self.reverse(): + if isinstance(scope, PartialLifetimeScope): + return scope + return None + + def get_current_class_scope(self) -> Optional[ClassScope]: + for scope in self.reverse(): + if isinstance(scope, ClassScope): + return scope + return None + + def get_current_scope_name(self) -> str: + return self.scopes[-1].name + + def is_inside_conditional_scope(self) -> bool: + for scope in self.reverse(): + if isinstance(scope, (ConditionalScope, LoopScope)): + return True + elif isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): + return False + assert False # GlobalScope is LifetimeScope + + def is_declaration_in_current_scope(self, name: str) -> bool: + return self.is_declaration_in_scope(name, self.scopes[-1]) + + def is_declaration_in_current_lifetime_scope(self, name: str) -> bool: + return self.is_declaration_in_scope(name, self.get_current_lifetime_scope()) + + def is_declaration_in_current_class(self, name: str) -> bool: + class_scope = self.get_current_class_scope() + if class_scope is not None: + return self.is_declaration_in_scope(name, class_scope) + return False + + def get_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.get_current_lifetime_scope(): + continue # members can be accessed via self, except from the current class scope + declaration = self.get_declaration_from_scope(name, scope) + if declaration is not None: + from cc_python_parser.variable_data import TypeVariableDeclaration + if isinstance(declaration, TypeVariableDeclaration): + pass # print("NEED FIX: TypeVariable") + return declaration + current_class_scope = self.get_current_class_scope() + if current_class_scope is not None: + for init_placeholder in current_class_scope.init_placeholders: + if name == init_placeholder.name: + return init_placeholder + return PlaceholderType(name) + + def get_variable_declaration(self, name: str) -> Union[VariableDeclaration, PlaceholderType]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.scopes[-1]: + continue + declaration = self.get_variable_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_function_declaration(self, name: str) -> Union[Declaration, PlaceholderType]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.scopes[-1]: + continue + declaration = self.get_function_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_class_declaration(self, name: str) -> Union[ClassDeclaration, PlaceholderType]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.scopes[-1]: + continue + declaration = self.get_class_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_declaration_from_member_access(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: + iter(iterator) + members = [next(iterator)] + # TODO: MethodVariable? + if isinstance(members[0], MemberAccessCollector.MethodData): + declaration = self.get_function_declaration(members[0].name) + elif isinstance(members[0], MemberAccessCollector.AttributeData): + declaration = self.get_variable_declaration(members[0].name) + else: + declaration = self.get_declaration(members[0].name) + if not isinstance(declaration, PlaceholderType): + return declaration + try: + module_variable = self.get_imported_module(iterator) + if module_variable is not None: + declaration = module_variable + else: + declaration_from_other_module = self.get_declaration_from_other_module(iterator) + if declaration_from_other_module is not None: + declaration = declaration_from_other_module + except StopIteration: + assert False + return declaration + + def get_scope_manager_from_member_access(self, iterator: MemberAccessCollectorIterator) -> Optional['ScopeManager']: + iter(iterator) + members = [next(iterator)] + if isinstance(members[0], MemberAccessCollector.MethodData): + declaration = self.get_function_declaration(members[0].name) + else: + declaration = self.get_declaration(members[0].name) + if not isinstance(declaration, PlaceholderType): + if isinstance(declaration, ImportedDeclaration): + return self.import_finder.get_scope_manager_by_location(declaration.module) + else: + return self + try: + module_import = self.get_import(iterator) + if module_import is not None: + return self.import_finder.get_scope_manager_by_location(module_import.location) + except StopIteration: + assert False + return None + + # import a.b.c -> c.f() vs a.b.c.f() + def get_declaration_from_other_module(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + # TODO: is this alias check enough? + if module.module.alias is None and len(module.path) >= len(iterator.mac.call_list): + continue + is_same = True + current = next(iterator) + name = current.name + iter(iterator) + if module.module.alias is not None: + is_same = module.module.alias == name + next(iterator) + else: + for i in range(0, len(module.path)): + name = next(iterator).name + if module.path[i] != name: + is_same = False + break + if is_same: + name = next(iterator).name + if module.is_module_import(): + other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) + if other_scope_manager is not None: + if isinstance(current, MemberAccessCollector.MemberData): + return other_scope_manager.get_function_declaration(name) + else: + return other_scope_manager.get_declaration(name) + for imp in module.imported: + if (imp.alias is not None and imp.alias == name) \ + or imp.name == name or imp.is_all_imported(): + other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) + if other_scope_manager is not None: + if isinstance(current, MemberAccessCollector.MemberData): + return other_scope_manager.get_function_declaration(name) + else: + return other_scope_manager.get_declaration(name) + return None + + def get_imported_module(self, iterator: MemberAccessCollectorIterator) -> Optional[ModuleVariableDeclaration]: + imported_module = None + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + if module.module.alias is None and len(module.path) != len(iterator.mac.call_list): + continue + if module.module.alias is not None: + if module.module.alias == iterator.mac.call_list[-1].name: + imported_module = module + break + else: + continue + else: + for i in range(0, len(module.path)): + next(iterator) + module_path = [m.name for m in iterator.mac.call_list] + module_path.reverse() + if module.path == module_path: + imported_module = module + break + if imported_module is not None and isinstance(scope, Scope): + for var in reversed(scope.variable_declarations): + if isinstance(var, ModuleVariableDeclaration) and \ + var.imported_module_location == imported_module.location: + return var + return None + + def get_import(self, iterator: MemberAccessCollectorIterator) -> Optional[Import]: + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + # TODO: is this alias check enough? + if module.module.alias is None and len(module.path) >= len(iterator.mac.call_list): + continue + current = next(iterator) + name = current.name + iter(iterator) + if module.module.alias is not None: + if module.module.alias == name: + return module + else: + for i in range(0, len(module.path)): + name = next(iterator).name + if module.path[i] != name: + return module + return None + + def get_declaration_from_current_class(self, name: str) -> Union[Declaration, PlaceholderType]: + declaration = self.get_declaration_from_scope(name, self.get_current_class_scope()) + if declaration is None: + return PlaceholderType(name) + return declaration + + def get_nonlocal_variable(self, name: str) -> Optional[VariableDeclaration]: + assert len(self.scopes) > 2 + for scope in self.scopes[-2:0:-1]: + var = scope.get_variable_by_name(name) + if var is not None: + return var + return None + + def get_global_variable(self, name: str) -> Optional[VariableDeclaration]: + return self.scopes[0].get_variable_by_name(name) + + @staticmethod + def get_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: + for declaration in reversed(scope.declarations): + if declaration.name == name: + return declaration + return None + + @staticmethod + def is_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return True + return False + + @staticmethod + def get_variable_declaration_from_scope(name: str, scope: Scope) -> Optional[VariableDeclaration]: + for declaration in reversed(scope.declarations): + if declaration.name != name: + continue + elif isinstance(declaration, VariableDeclaration): + return declaration + elif isinstance(declaration, FunctionDeclaration) and not isinstance(declaration, ClassInitDeclaration): + return FunctionVariableDeclaration(declaration) + elif isinstance(declaration, ClassDeclaration): + return TypeVariableDeclaration(declaration) + return None + + @staticmethod + def get_function_declaration_from_scope(name: str, scope: Scope) \ + -> Optional[Union[FunctionDeclaration, ClassDeclaration, FunctionVariableDeclaration]]: + for declaration in reversed(scope.declarations): + if declaration.name != name: + continue + elif isinstance(declaration, (FunctionDeclaration, ClassDeclaration)): + return declaration + elif isinstance(declaration, ClassDeclaration): + return ClassInitDeclaration(declaration) + else: + function_variables = [t for t in declaration.type if isinstance(t, FunctionVariableDeclaration)] + if len(function_variables) > 0: + return function_variables[0] + else: + return declaration + return None + + @staticmethod + def get_class_declaration_from_scope(name: str, scope: Scope) -> Optional[ClassDeclaration]: + for declaration in reversed(scope.class_declarations): + if declaration.name == name: + return declaration + return None + + def get_current_class_declaration(self) -> Optional[ClassDeclaration]: + current_class_scope_found = False + for scope in self.reverse(): + if current_class_scope_found and isinstance(scope, LifetimeScope) and \ + not isinstance(scope, PartialLifetimeScope): + return scope.class_declarations[-1] + elif not current_class_scope_found: + current_class_scope_found = isinstance(scope, ClassScope) + return None + + def set_global_import_table(self, import_table: ImportTable): + global_scope = self.scopes[0] + if isinstance(global_scope, GlobalScope): + global_scope.import_table = import_table + + def append_import(self, node: ast.Import): + for scope in self.reverse(): + if isinstance(scope, ImportScope): + scope.import_table.append_import(node, not isinstance(scope, GlobalScope)) + return + assert False + + def append_import_from(self, node: ast.ImportFrom): + for scope in self.reverse(): + if isinstance(scope, ImportScope): + scope.import_table.append_import_from(node, not isinstance(scope, GlobalScope)) + return + assert False + + def handle_import_scope(self): + if isinstance(self.scopes[-1], ImportScope): + for declaration in self.scopes[-1].declarations: + if isinstance(declaration, ImportedDeclaration): + declaration.imported_declaration.usages.extend(declaration.usages) + + def persist_current_scope(self): + # for declaration in self.scopes[-1].declarations: + # print(self.get_qualified_name_from_current_scope(declaration)) + + if self.persist_symbols > 0: + return + + self.handle_import_scope() + + import_dto = ImportDTO(self.current_file) + + for var in self.scopes[-1].variable_declarations: + if isinstance(var, ImportedDeclaration): + import_dto.add_symbol_import(var.qualified_name, var.module.location, var.position, + var.imported_declaration.qualified_name) + elif isinstance(var, ModuleVariableDeclaration): + import_dto.add_module_import(var.qualified_name, var.imported_module_location, var.file_position) + else: + self.persistence.persist_variable(var.create_dto()) + + for func in self.scopes[-1].function_declarations: + if isinstance(func, ImportedDeclaration): + import_dto.add_symbol_import(func.qualified_name, func.module.location, func.position, + func.imported_declaration.qualified_name) + else: + self.persistence.persist_function(func.create_dto()) + + for cl in self.scopes[-1].class_declarations: + if isinstance(cl, ImportedDeclaration): + import_dto.add_symbol_import(cl.qualified_name, cl.module.location, cl.position, + cl.imported_declaration.qualified_name) + else: + self.persistence.persist_class(cl.create_dto()) + + if len(import_dto.imported_modules) > 0 or len(import_dto.imported_symbols) > 0: + self.persistence.persist_import(import_dto) + + def get_qualified_scope_name_from_current_scope(self, scopes: Optional[List[Scope]] = None) -> str: + qualified_name_parts = [] + if scopes is None: + qualified_name_parts.extend(self.get_qualified_scope_name_from_scopes(self.scopes)) + else: + qualified_name_parts.extend(self.get_qualified_scope_name_from_scopes(scopes)) + file_name = self.current_file.name + assert file_name[-3:] == '.py' + qualified_name_parts.append(file_name[:-3]) + directory = self.current_file.parent + while True: + init_file_location = Path(directory).joinpath("__init__.py") + if init_file_location.exists(): + qualified_name_parts.append(directory.name) + directory = directory.parent + else: + qualified_name_parts.append(str(hash(directory))) + break + return '.'.join(reversed(qualified_name_parts)) + + def get_qualified_scope_name_from_scopes(self, scopes: List[Scope]): + qualified_name_parts = [] + for scope in reversed(scopes): + if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope + qualified_name_parts.append(scope.name) + return qualified_name_parts + + def get_qualified_name_from_current_scope(self, declaration_name: str, line_num: int) -> str: + return self.get_qualified_scope_name_from_current_scope() + '.' + declaration_name + ':' + str(line_num) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/skip_file.py b/plugins/python/parser/src/scripts/cc_python_parser/skip_file.py new file mode 100644 index 000000000..2d1449f9d --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/skip_file.py @@ -0,0 +1,46 @@ +import json +import re +import os +from json import JSONDecodeError +from typing import Final +from pathlib import Path + +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.parse_exception import ParseException + +SKIP_FILE_NAME: Final[str] = "skip.json" + + +def read_skip_file(persistence: ModelPersistence) -> ParseException: + """ + Reads regexes to skip files and directories from 'skip.json' while parsing, if it exists: + { + "files": [], + "directories": [ + "^[.]", + "^venv$" + ] + } + """ + + ff = lambda p: False + dd = lambda p: False + + split_path = __file__.split(os.sep) + del split_path[-1] + split_path[-1] = SKIP_FILE_NAME + path = f"{os.sep}{os.path.join(*split_path)}" + + if Path(path).exists(): + with open(path, 'r') as config_file: + try: + skip = json.load(config_file) + if skip['files']: + ff = lambda p: any(re.search(f, p) is not None for f in skip['files']) + if skip['directories']: + dd = lambda p: any(re.search(d, p) is not None for d in skip['directories']) + except JSONDecodeError as e: + persistence.log_warning(f"Invalid JSON file: {SKIP_FILE_NAME} (line: {e.lineno}, column: {e.colno})") + except Exception as e: + persistence.log_warning(f"Error during {SKIP_FILE_NAME} decoding: {e}") + return ParseException(dd, ff) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py new file mode 100644 index 000000000..e902d705b --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py @@ -0,0 +1,1130 @@ +import ast +from pathlib import PurePath +from typing import List, Optional, Any, Union, Set, TypeVar + +from cc_python_parser.built_in_functions import get_built_in_function +from cc_python_parser.built_in_types import GenericType, BuiltIn, List as BuiltInList +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.placeholder_function_declaration_cache import PlaceholderFunctionDeclarationCache +from cc_python_parser.symbol_collector_interface import SymbolCollectorBase +from cc_python_parser.base_data import Declaration, Usage, ImportedDeclaration +from cc_python_parser.class_data import ClassDeclaration, ImportedClassDeclaration +from cc_python_parser.class_preprocessor import PreprocessedClass +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.import_preprocessor import ImportTable +from cc_python_parser.logger import logger +from cc_python_parser.member_access_collector import MemberAccessCollector, MemberAccessCollectorIterator +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.preprocessed_function import PreprocessedFunction +from cc_python_parser.preprocessed_variable import PreprocessedVariable +from cc_python_parser.scope import ClassScope, FunctionScope, GeneratorScope, LambdaScope, \ + ExceptionScope, ConditionalScope, LoopScope, GlobalScope +from cc_python_parser.function_data import FunctionDeclaration, ImportedFunctionDeclaration, \ + StaticFunctionDeclaration +from cc_python_parser.scope_manager import ScopeManager +from cc_python_parser.symbol_finder import SymbolFinder +from cc_python_parser.type_data import PlaceholderType, InitVariablePlaceholderType, VariablePlaceholderType +from cc_python_parser.type_deduction import TypeDeduction +from cc_python_parser.type_hint_data import TypeHintType +from cc_python_parser.variable_data import VariableDeclaration, ImportedVariableDeclaration, \ + ModuleVariableDeclaration, StaticVariableDeclaration, NonlocalVariableDeclaration, GlobalVariableDeclaration, \ + FunctionVariableDeclaration, TypeVariableDeclaration, MethodVariableDeclaration +from cc_python_parser.common.metrics import metrics + + +class SymbolCollector(ast.NodeVisitor, SymbolFinder, SymbolCollectorBase): + def __init__(self, root, current_file: PurePath, preprocessed_file: PreprocessedFile, + import_finder: ImportFinder, persistence: ModelPersistence, + function_symbol_collector_factory: FunctionSymbolCollectorFactory): + SymbolCollectorBase.__init__(self, preprocessed_file, import_finder) + self.root = root + self.current_file = current_file + self.scope_manager: ScopeManager = ScopeManager(current_file, import_finder, persistence) + self.persistence: ModelPersistence = persistence + self.current_function_declaration: List[FunctionDeclaration] = [] + self.current_class_declaration: List[ClassDeclaration] = [] + self.function_symbol_collector_factory = function_symbol_collector_factory + self.type_deduction = TypeDeduction(self, self.scope_manager, + preprocessed_file, self.function_symbol_collector_factory, self.persistence) + + def collect_symbols(self): + self.visit(self.root) + self.post_process() + + def generic_visit(self, node: ast.AST) -> Any: + if type(self) is SymbolCollector: + metrics.add_ast_count() + ast.NodeVisitor.generic_visit(self, node) + + def visit_Import(self, node: ast.Import) -> Any: + metrics.add_import_count() + modules = ImportTable.convert_ast_import_to_import(node) + for module in modules: + if not module.is_module_import(): + continue + # scope.imports.append((module, self.import_finder.get_global_scope_by_location(module.location))) + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(module.module.name, + r.start_position.line) + self.scope_manager.append_module_variable_to_current_scope( + ModuleVariableDeclaration(qualified_name, module.path[-1], module.location, + FilePosition(self.current_file, module.range), + self.import_finder.get_global_scope_by_location(module.location))) + self.scope_manager.append_import(node) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: + metrics.add_import_count() + modules = ImportTable.convert_ast_import_from_to_import(node) + for module in modules: + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(module.module.name, + r.start_position.line) + self.scope_manager.append_module_variable_to_current_scope( + ModuleVariableDeclaration(qualified_name, module.path[-1], module.location, + FilePosition(self.current_file, module.range), + self.import_finder.get_global_scope_by_location(module.location))) + if not module.is_module_import(): + scope_manager: ScopeManager = self.import_finder.get_scope_manager_by_location(module.location) + if scope_manager is None: + # import from library + continue + # in case of circular import this case is not valid (runtime error) -> must find declaration + file_position = FilePosition(self.current_file, create_range_from_ast_node(node)) + # TODO: handle transitive imports + for imported in module.imported: + if imported.is_all_imported(): + for declaration in scope_manager.get_global_scope().declarations: + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(declaration.name, + r.start_position.line) + if isinstance(declaration, VariableDeclaration): + var = ImportedVariableDeclaration(qualified_name, + declaration.name, file_position, declaration, module) + self.scope_manager.append_variable_to_current_scope(var) + self.imported_declaration_scope_map[var] = scope_manager + elif isinstance(declaration, FunctionDeclaration): + func = ImportedFunctionDeclaration(qualified_name, + declaration.name, file_position, declaration, module) + self.scope_manager.append_function_to_current_scope(func) + self.imported_declaration_scope_map[func] = scope_manager + elif isinstance(declaration, ClassDeclaration): + c = ImportedClassDeclaration(qualified_name, + declaration.name, file_position, declaration, module) + self.scope_manager.append_class_to_current_scope(c) + self.imported_declaration_scope_map[c] = scope_manager + else: + assert False + else: + # TODO: redefinition? + declaration = scope_manager.get_declaration(imported.name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(imported.name, + r.start_position.line) + if isinstance(declaration, PlaceholderType): + continue + elif isinstance(declaration, VariableDeclaration): + var = ImportedVariableDeclaration(qualified_name, + imported.name if imported.alias is None else imported.alias, + file_position, declaration, module) + self.scope_manager.append_variable_to_current_scope(var) + self.imported_declaration_scope_map[var] = scope_manager + elif isinstance(declaration, FunctionDeclaration): + func = ImportedFunctionDeclaration(qualified_name, + imported.name if imported.alias is None else imported.alias, + file_position, declaration, module) + self.scope_manager.append_function_to_current_scope(func) + self.imported_declaration_scope_map[func] = scope_manager + elif isinstance(declaration, ClassDeclaration): + c = ImportedClassDeclaration(qualified_name, + imported.name if imported.alias is None else imported.alias, + file_position, declaration, module) + self.scope_manager.append_class_to_current_scope(c) + self.imported_declaration_scope_map[c] = scope_manager + else: + assert False + self.scope_manager.append_import_from(node) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> Any: + metrics.add_class_count() + base_classes = self.collect_base_classes(node) + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name, r.start_position.line) + new_class = ClassDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), + base_classes, self.get_documentation(node)) + self.class_def_process_derived_members(new_class, base_classes) + # TODO: handle preprocessed classes + self.scope_manager.append_class_to_current_scope(new_class) + self.current_class_declaration.append(new_class) + self.scope_manager.persistence.persist_preprocessed_class(new_class.create_dto()) + + # TODO: check if has at least 1 parameter? (self) + # TODO: range could be the __init__ + # self.scope_manager.append_function_to_current_scope(ClassInitDeclaration(new_class)) + + def action(): + self.generic_visit(node) + scope = self.scope_manager.get_current_scope() + assert isinstance(scope, ClassScope) + for var in scope.variable_declarations: + if isinstance(var, StaticVariableDeclaration) and var not in new_class.static_attributes: + new_class.static_attributes.append(var) + elif isinstance(var, VariableDeclaration) and var not in new_class.attributes: + new_class.attributes.append(var) + for func in scope.function_declarations: + if isinstance(func, StaticFunctionDeclaration): + new_class.static_methods.append(func) + elif isinstance(func, FunctionDeclaration): + new_class.methods.append(func) + else: + assert False + for c in scope.class_declarations: + new_class.classes.append(c) + + self.scope_manager.with_scope(ClassScope(node.name, + self.scope_manager.get_qualified_scope_name_from_current_scope(), + new_class.file_position), action) + self.class_def_post_process(new_class) + del self.current_class_declaration[-1] + + def collect_base_classes(self, node: ast.ClassDef) -> List[ClassDeclaration]: + base_classes = [] + for base_class in node.bases: + base_class_types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(base_class)) + if len(base_class_types) > 1: + continue # TODO: eg: standard Lib > ctypes > test > test_byteswap.py > function 'test_struct_struct' + for base_class_type in base_class_types: + if isinstance(base_class_type, ClassDeclaration): + base_classes.append(base_class_type) + elif isinstance(base_class_type, TypeVariableDeclaration): + if isinstance(base_class_type.reference, ImportedDeclaration): + base_classes.append(base_class_type.reference.imported_declaration) + elif isinstance(base_class_type.reference, ClassDeclaration): + base_classes.append(base_class_type.reference) + else: + pass # print('') + else: + pass # print('') + return base_classes + + def class_def_process_derived_members(self, cl: ClassDeclaration, base_classes: List[ClassDeclaration]): + declaration_type = TypeVar('declaration_type', bound=Declaration) + + def collect_derived_members(members: List[List[declaration_type]]) -> List[declaration_type]: + declarations = [] + for base_class_members in members: + for member in base_class_members: + if all([m.name != member.name for m in declarations]): + declarations.append(member) + return declarations + + cl.attributes.extend(collect_derived_members([d.attributes for d in base_classes])) + cl.static_attributes.extend(collect_derived_members([d.static_attributes for d in base_classes])) + cl.methods.extend(collect_derived_members([d.methods for d in base_classes])) + cl.static_methods.extend(collect_derived_members([d.static_methods for d in base_classes])) + cl.classes.extend(collect_derived_members([d.classes for d in base_classes])) + + def class_def_post_process(self, cl: ClassDeclaration): + def remove_hidden_declarations(declarations: List[Declaration]): + for declaration in reversed(declarations): + hidden_declarations = [d for d in declarations if d.name == declaration.name and d is not declaration] + for hidden_declaration in hidden_declarations: + declarations.remove(hidden_declaration) + + remove_hidden_declarations(cl.attributes) + remove_hidden_declarations(cl.static_attributes) + remove_hidden_declarations(cl.methods) # TODO: singledispatchmethod + remove_hidden_declarations(cl.static_methods) + remove_hidden_declarations(cl.classes) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + self.visit_common_function_def(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: + self.visit_common_function_def(node) + + def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: + metrics.add_function_count() + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name, r.start_position.line) + if self.is_static_method(node): + new_function = StaticFunctionDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), + [], self.get_documentation(node)) + else: + new_function = FunctionDeclaration(node.name, qualified_name, + FilePosition(self.current_file, r), [], self.get_documentation(node)) + self.scope_manager.append_function_to_current_scope(new_function) + self.current_function_declaration.append(new_function) + + # if node.name == '__init__' and isinstance(self.scopes[-1], ClassScope): # ctor ~ __init__ + + def action(): + # self.collect_parameters(node) + self.process_function_def(node) + if any(isinstance(x, PlaceholderType) for x in self.current_function_declaration[-1].type): + is_method = self.scope_manager.is_current_scope_method() + func_declaration_data = PlaceholderFunctionDeclarationCache.FunctionDeclarationData(node, is_method) + self.scope_manager.placeholder_function_declaration_cache. \ + add_function_declaration(new_function, func_declaration_data) + + self.scope_manager.with_scope(FunctionScope(node.name, + self.scope_manager.get_qualified_scope_name_from_current_scope(), + new_function.file_position), action) + del self.current_function_declaration[-1] + + def get_documentation(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef]) -> str: + documentation = ast.get_docstring(node) + if documentation is None: + return "" + else: + return documentation + + def process_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: + is_init = self.scope_manager.is_current_scope_init() + is_method = self.scope_manager.is_current_scope_method() + + def get_first_param() -> Optional[ast.arg]: + if len(node.args.posonlyargs) > 0: + return node.args.posonlyargs[0] + if len(node.args.args) > 0: + return node.args.args[0] + if node.args.vararg is not None: + return node.args.vararg + if len(node.args.kwonlyargs) > 0: + return node.args.kwonlyargs[0] + if node.args.kwarg is not None: + return node.args.kwarg + return None + + def get_annotation_type(annotation: Optional[ast.expr]) -> Optional[TypeHintType]: + if annotation is None: + return None + + hinted_type = None + if annotation is not None: + if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): + hinted_type = None + else: + t = self.type_deduction.deduct_type(annotation) + current_type = self.type_deduction.get_current_type(t) + assert len(current_type) == 0 or len(current_type) == 1 + if len(current_type) == 1: + tt = list(current_type)[0] + if isinstance(tt, (ClassDeclaration, BuiltIn)): + hinted_type = TypeHintType(tt) + elif isinstance(tt, TypeVariableDeclaration): + if isinstance(tt.reference, (ClassDeclaration, BuiltIn)): + hinted_type = TypeHintType(tt.reference) + elif isinstance(tt.reference, ImportedDeclaration): + hinted_type = TypeHintType(tt.reference.imported_declaration) + else: + hinted_type = None + else: + hinted_type = None + return hinted_type + + def process_arg(param: Optional[ast.arg]): + if param is None: + return + + r = create_range_from_ast_node(param) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(param.arg, r.start_position.line) + new_variable = VariableDeclaration(param.arg, qualified_name, FilePosition(self.current_file, r)) + + hinted_type = get_annotation_type(param.annotation) + + if param.arg == 'self' and is_method and get_first_param() is param: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + elif is_init: + if hinted_type is not None: + new_variable.type.add(hinted_type) + else: + init_var_placeholder_type = InitVariablePlaceholderType(param.arg) + new_variable.type.add(init_var_placeholder_type) + self.scope_manager.get_current_class_scope().init_placeholders.append(init_var_placeholder_type) + else: + if hinted_type is not None: + new_variable.type.add(hinted_type) + else: + new_variable.type.add(VariablePlaceholderType(param.arg)) + self.scope_manager.append_variable_to_current_scope(new_variable) + self.current_function_declaration[-1].parameters.append(new_variable) + + for arg in node.args.args: # simple arguments + process_arg(arg) + process_arg(node.args.vararg) # *arg + for arg in node.args.kwonlyargs: # arguments between vararg and kwarg + process_arg(arg) + process_arg(node.args.kwarg) # **arg + for default in node.args.defaults: # last N args default value + self.visit(default) + for default in node.args.kw_defaults: # default values of kwonlyargs (all) + if default is not None: + self.visit(default) + for arg in node.args.posonlyargs: # arguments before / "parameter" (after -> args) + process_arg(arg) + + for decorator in node.decorator_list: + self.visit(decorator) + + if node.returns is not None: + self.visit(node.returns) + + for stmt in node.body: + self.visit(stmt) + + self.current_function_declaration[-1].local_variables.extend( + self.scope_manager.get_current_scope().variable_declarations) + + return_type = get_annotation_type(node.returns) + if return_type is not None: + self.current_function_declaration[-1].type.add(return_type) + + def visit_Return(self, node: ast.Return) -> Any: + self.visit_common_return(node) + + def visit_Yield(self, node: ast.Yield) -> Any: + if isinstance(self.scope_manager.get_current_scope(), LambdaScope): + self.generic_visit(node) + else: + self.visit_common_return(node) + + def visit_YieldFrom(self, node: ast.YieldFrom) -> Any: + self.visit_common_return(node) + + def visit_common_return(self, node: (ast.Return, ast.Yield, ast.YieldFrom)): + assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) + if node.value is not None: + types = self.type_deduction.deduct_type(node) + if any(t is self.current_function_declaration[-1] for t in types): + types = {t for t in types if t is not self.current_function_declaration[-1]} + if len(types) == 0: + self.current_function_declaration[-1].type.add(VariablePlaceholderType('')) + else: + self.current_function_declaration[-1].type.update(types) + self.generic_visit(node) + + def visit_NamedExpr(self, node: ast.NamedExpr) -> Any: + self.handle_assignment(node.target, node.value) + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> Any: + for target in node.targets: + self.handle_assignment(target, node.value) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> Any: + self.handle_assignment(node.target, node.value) + self.generic_visit(node) + + def handle_assignment(self, target: ast.AST, value: ast.AST): + if isinstance(target, (ast.Name, ast.Attribute, ast.Subscript, ast.Starred)): + self.handle_single_assignment(target, value) + elif isinstance(target, (ast.List, ast.Tuple)): + if isinstance(value, (ast.List, ast.Tuple)): + assert len(target.elts) == len(value.elts) + for i in range(0, len(target.elts)): + self.handle_assignment(target.elts[i], value.elts[i]) + else: + for t in target.elts: + self.handle_assignment(t, value) + else: + assert False + + def handle_single_assignment(self, target: ast.AST, value: ast.AST): + mac = None + if isinstance(target, (ast.Attribute, ast.Subscript, ast.Starred)): + mac = MemberAccessCollector(target, self.persistence) + name = mac.call_list[0].name + elif isinstance(target, ast.Name): + name = target.id + else: + assert False + + if name == '_': + return # TODO + + if value is None: # variable: type (no value!) + types = set() + else: + types = self.type_deduction.get_current_type(self.type_deduction.deduct_type(value)) + + if isinstance(target, ast.Subscript): + self.handle_subscript_assignment(mac, types) + return + + r = create_range_from_ast_node(target) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + + if self.is_static_variable(target): + new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + else: + new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + + if self.scope_manager.is_current_scope_init() and mac is not None and len(mac.call_list) == 2 and \ + mac.call_list[-1].name == 'self': + if not self.scope_manager.is_declaration_in_current_class(new_variable.name): + self.scope_manager.append_variable_to_current_class_scope(new_variable) + self.scope_manager.get_current_class_declaration().attributes.append(new_variable) + elif isinstance(target, (ast.Name, ast.Starred)): + if self.scope_manager.is_inside_conditional_scope() and \ + not isinstance(declaration := self.scope_manager.get_declaration(name), PlaceholderType): + if isinstance(value, (ast.Name, ast.Call, ast.Attribute)): + d = self.get_declaration(value) + dd = self.get_declaration(target) + if self.has_recursion_in_type(declaration, types): + declaration.type = self.fix_recursion_in_type(declaration, types) + else: + declaration.type.update(new_variable.type) + declaration.type = \ + {t for t in self.type_deduction.get_current_type(declaration.type) + if not isinstance(t, PlaceholderType)} + else: + if isinstance(value, (ast.Name, ast.Call, ast.Attribute)): + d = self.get_declaration(value) + if d is not None: + new_variable.type = \ + {t for t in self.type_deduction.get_current_type(d) if not isinstance(t, PlaceholderType)} + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + pass + + def has_recursion_in_type(self, var: Optional[Union[VariableDeclaration, PlaceholderType]], types: Set): + if var is None or isinstance(var, PlaceholderType): + return False + for t in types: + if var is t: + return True + elif isinstance(t, GenericType) and self.has_recursion_in_type(var, t.types): + return True + return False + + def fix_recursion_in_type(self, var: Optional[Union[VariableDeclaration, PlaceholderType]], types: Set) -> Set: + if var is None or isinstance(var, PlaceholderType): + return set() + fixed_types = set() + for t in types: + if var is t: + if isinstance(var, GenericType): + fixed_types.update(var.types) + else: + fixed_types.update(var.type) + elif isinstance(t, GenericType): + fixed_types.update(self.fix_recursion_in_type(var, t.types)) + return fixed_types + + def handle_subscript_assignment(self, mac: MemberAccessCollector, types: Set): + name = mac.call_list[0].name + if name is not None: + declaration = self.scope_manager.get_variable_declaration(name) + if not isinstance(declaration, PlaceholderType): + if self.has_recursion_in_type(declaration, types): + declaration.types = self.fix_recursion_in_type(declaration, types) + elif generics := [d for d in declaration.type if isinstance(d, GenericType)]: + for generic in generics: + if self.has_recursion_in_type(generic, types): + generic.types = self.fix_recursion_in_type(generic, types) + else: + generic.types.update(types) + elif len(mac.call_list) == 1 and isinstance(mac.call_list[0], MemberAccessCollector.ReturnValueData) and \ + isinstance(mac.call_list[0].node, ast.BoolOp): + pass # TODO: (list_1 or list_2)[i] = x, eg: Lib > idlelib > query.py > showerror function + else: # eg.: a[i][j] = k + depth = 1 + for i in range(1, len(mac.call_list)): + if not isinstance(mac.call_list[i], MemberAccessCollector.ReturnValueData): + depth = i + break + declarations = {self.scope_manager.get_variable_declaration(mac.call_list[depth].name)} + generic_types = set() + for i in range(depth, 0, -1): + for declaration in declarations: + if not isinstance(declaration, PlaceholderType) and \ + (generics := [d for d in declaration.type if isinstance(d, GenericType)]): + for generic in generics: + if self.has_recursion_in_type(generic_types, generic.types): + generic_types = self.fix_recursion_in_type(generic_types, generic.types) + else: + generic_types.update(generic.types) + declarations = generic_types.copy() + generic_types.clear() + for declaration in declarations: + if not isinstance(declaration, PlaceholderType) and \ + (generics := [d for d in declaration.type if isinstance(d, GenericType)]): + for generic in generics: + if self.has_recursion_in_type(generic, types): + generic.types = self.fix_recursion_in_type(generic, types) + else: + generic.types.update(types) + + def visit_For(self, node: ast.For) -> Any: + self.visit_common_for(node) + + def visit_AsyncFor(self, node: ast.AsyncFor) -> Any: + self.visit_common_for(node) + + def visit_common_for(self, node: Union[ast.For, ast.AsyncFor]): + def handle_loop_variable(types, var): + declarations = self.get_declaration(var) + for declaration in declarations: + if declaration is None or isinstance(declaration, PlaceholderType): + assert isinstance(var, ast.Name) + if var.id == '_': + continue # TODO + r = create_range_from_ast_node(var) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(var.id, r.start_position.line) + new_variable = VariableDeclaration(var.id, qualified_name, + FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + tt = types + if self.has_recursion_in_type(declaration, tt): + tt = self.fix_recursion_in_type(declaration, tt) + declaration.type = self.type_deduction.get_current_type(tt) + if len(declarations) == 0: + self.handle_new_variable(var) + + def handle_loop_target(types, elem): + if isinstance(elem, (ast.Name, ast.Starred)): + handle_loop_variable(types, elem) + elif isinstance(elem, (ast.Tuple, ast.List)): + for var in elem.elts: + handle_loop_target(types, var) + else: + assert False + + def action(): + types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(node.iter)) + if len(types) == 0: + self.handle_new_variable(node.target) + + for t in types: + if isinstance(t, GenericType): + handle_loop_target(t.types, node.target) + elif not isinstance(t, PlaceholderType): + iterable = self.get_iterable_type({t}) + if iterable is not None: + next_obj = self.get_next_type(iterable) + if next_obj is not None: + handle_loop_target(next_obj, node.target) + else: + self.handle_new_variable(node.target) + else: + self.handle_new_variable(node.target) + else: + self.handle_new_variable(node.target) + self.generic_visit(node) + + self.scope_manager.with_scope(LoopScope(), action) + + def handle_new_variable(self, node): + if isinstance(node, (ast.Name, ast.Starred)): + r = create_range_from_ast_node(node) + if isinstance(node, ast.Name): + id = node.id + elif isinstance(node, ast.Starred): + id = node.value.id + else: + assert False + if id == '_': + return # TODO + file_position = FilePosition(self.current_file, r) + declaration = self.scope_manager.get_declaration(id) + if not isinstance(declaration, PlaceholderType) and declaration.file_position == file_position: + return + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(id, r.start_position.line) + new_variable = VariableDeclaration(id, qualified_name, file_position) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(node, (ast.Tuple, ast.List)): + for var in node.elts: + self.handle_new_variable(var) + elif isinstance(node, (ast.Attribute, ast.Subscript)): + pass # TODO: handle subscript as loop variable + else: + assert False, f"Unhandled type: {type(node)}" + + def get_iterable_type(self, t: Set) -> Optional[Set]: + current_type = self.type_deduction.get_current_type(t) + for ct in current_type: + if isinstance(ct, ClassDeclaration): + for m in reversed(ct.methods): + if m.name == '__iter__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + for m in reversed(ct.static_methods): + if m.name == '__iter__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + return None + + def get_next_type(self, t: Set) -> Optional[Set]: + current_type = self.type_deduction.get_current_type(t) + for ct in current_type: + if isinstance(ct, ClassDeclaration): + for m in reversed(ct.methods): + if m.name == '__next__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + for m in reversed(ct.static_methods): + if m.name == '__next__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + return None + + def visit_With(self, node: ast.With) -> Any: + self.common_with_visit(node) + + def visit_AsyncWith(self, node: ast.AsyncWith) -> Any: + self.common_with_visit(node) + + def common_with_visit(self, node: Union[ast.With, ast.AsyncWith]) -> Any: + def handle_vars(variables, types): + if isinstance(variables, ast.Name): + if variables.id == '_': + return # TODO + r = create_range_from_ast_node(variables) + qn = self.scope_manager.get_qualified_name_from_current_scope(variables.id, r.start_position.line) + new_variable = VariableDeclaration(variables.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(variables, (ast.List, ast.Tuple)): + for elem in variables.elts: + handle_vars(elem, types) + elif isinstance(variables, (ast.Attribute, ast.Subscript)): + pass # TODO: handle ast.Attribute and ast.Subscript eg: Lib > test > test_buffer.py + elif isinstance(variables, ast.Starred): + r = create_range_from_ast_node(variables) + t = {BuiltInList(types)} + qn = self.scope_manager.get_qualified_name_from_current_scope(variables.value.id, r.start_position.line) + new_variable = VariableDeclaration(variables.value.id, qn, FilePosition(self.current_file, r), t) + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + assert False + + def action(): + for with_item in node.items: + if with_item.optional_vars is not None: + types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(with_item.context_expr)) + handle_vars(with_item.optional_vars, types) + self.generic_visit(node) + + self.scope_manager.with_scope(LoopScope(), action) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> Any: + def action(): + if node.name is not None and node.name != '_': # TODO + r = create_range_from_ast_node(node) + types = self.type_deduction.deduct_type(node.type) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(node.name, r.start_position.line) + new_variable = VariableDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), types) + self.scope_manager.append_exception_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(ExceptionScope(), action) + + def visit_Global(self, node: ast.Global) -> Any: + r = create_range_from_ast_node(node) + for name in node.names: + reference = self.scope_manager.get_global_variable(name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + if reference is None: + reference = self.scope_manager.get_global_variable(name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + if reference is None: + v = VariableDeclaration(name, qualified_name, FilePosition.get_empty_file_position(), + {VariablePlaceholderType(name)}) + # TODO: global variable not found, eg: Lib > test > test_dummy_threading.py > global running + self.scope_manager.append_variable_to_current_scope( + GlobalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), v)) + continue + self.scope_manager.append_variable_to_current_scope( + GlobalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) + if reference is not None: + reference.usages.append(Usage(name, FilePosition(self.current_file, r))) + + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_ListComp(self, node: ast.ListComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_SetComp(self, node: ast.SetComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_DictComp(self, node: ast.DictComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_Lambda(self, node: ast.Lambda) -> Any: + def action(): + for var in node.args.args: + if var.arg == '_': + continue # TODO + r = create_range_from_ast_node(var) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(var.arg, r.start_position.line) + new_variable = VariableDeclaration(var.arg, qualified_name, FilePosition(self.current_file, r)) + self.scope_manager.append_variable_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(LambdaScope(), action) + + def visit_comprehension(self, node: ast.comprehension) -> Any: + def handle_comprehension_target(target): + if isinstance(target, ast.Name): + if target.id == '_': + return + r = create_range_from_ast_node(target) + qn = self.scope_manager.get_qualified_name_from_current_scope(target.id, r.start_position.line) + types = self.type_deduction.get_current_type(self.type_deduction.deduct_type(node.iter)) + new_variable = VariableDeclaration(target.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(target, ast.Tuple): + for var in target.elts: + handle_comprehension_target(var) + elif isinstance(target, ast.Subscript): + pass # TODO: handle subscript as comprehension target + else: + assert False, 'Comprehension target type not handled: ' + type(target) + + handle_comprehension_target(node.target) + self.generic_visit(node) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> Any: + r = create_range_from_ast_node(node) + for name in node.names: + reference = self.scope_manager.get_nonlocal_variable(name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + if reference is None: + v = VariableDeclaration(name, qualified_name, FilePosition.get_empty_file_position(), + {VariablePlaceholderType(name)}) + # TODO: nonlocal variable declared after function declaration + self.scope_manager.append_variable_to_current_scope( + NonlocalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), v)) + continue + self.scope_manager.append_variable_to_current_scope( + NonlocalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) + if reference is not None: + reference.usages.append(Usage(name, FilePosition(self.current_file, r))) + + def visit_Name(self, node: ast.Name) -> Any: + if isinstance(node.ctx, ast.Store): + var = self.get_declaration(node) + if var is not None and not isinstance(list(var)[0], PlaceholderType): + other = self.create_var_usage(node.id, node) + if not list(var)[0].is_same_usage(other): + self.append_variable_usage(node.id, node) + elif node.id == '_': + pass + else: + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(node.id, r.start_position.line) + self.scope_manager.append_variable_to_current_scope( + VariableDeclaration(node.id, qualified_name, FilePosition(self.current_file, r))) + assert False, f'Assignment target, was not handled at {str(r)} in {self.current_file}' + elif isinstance(node.ctx, (ast.Load, ast.Del)): + self.append_variable_usage(node.id, node) + else: + assert False, "Unknown context" + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> Any: + self.append_variable_usage(node.attr, node) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> Any: + self.visit(node.func) + + for arg in node.args: + self.visit(arg) + for keyword in node.keywords: + self.visit(keyword.value) + + self.append_function_usage(node) + + def visit_Delete(self, node: ast.Delete) -> Any: + # TODO: remove variable (if not -> only runtime error) + self.generic_visit(node) + + def visit_If(self, node: ast.If) -> Any: + # TODO: can be recursive (eg. if..elif..elif..else..), is it a problem? + self.scope_manager.with_scope(ConditionalScope(), lambda: self.generic_visit(node)) + + def append_variable_usage(self, name: str, node: (ast.Name, ast.Attribute, ast.Subscript)) -> None: + usage = self.create_var_usage(name, node) + variable_declarations = self.get_declaration(node) + for var in variable_declarations: + if var is not None and not isinstance(var, PlaceholderType): + if isinstance(var, ImportedDeclaration): + var.imported_declaration.usages.append(usage) + else: + var.usages.append(usage) + logger.debug('Var Usage: ' + str(usage) + ' ' + var.get_type_repr()) + return + # TODO: annotations -> assert + assert True, "Variable not found: " + usage.name + " (usage: line - " + str(usage.file_position) + ")" + + def append_function_usage(self, node: ast.Call) -> None: + usage = self.create_function_usage(node) + mac = self.member_access_collector_type(node, self.persistence) + is_method_call = (len(mac.call_list) == 2 and + isinstance(mac.call_list[1], MemberAccessCollector.AttributeData) and + mac.call_list[1].name == 'self') + if not (len(mac.call_list) == 1 or is_method_call): + # functions from another module and modules + # return # TODO: handle these cases (type deduction) + pass + function_declarations = self.get_declaration(node) + if function_declarations is not None: + for func in function_declarations: + if func is not None and not isinstance(func, PlaceholderType): + if isinstance(func, ImportedDeclaration): + func.imported_declaration.usages.append(usage) + else: + func.usages.append(usage) + logger.debug('Func Usage: ' + str(usage) + ' ' + func.get_type_repr()) + return + if self.is_builtin(usage.name): + self.append_builtin_usage(usage) + return + # TODO: imported functions/methods from libraries not in the project (builtins!) + assert True, "Function not found: " + usage.name + " (usage: start position - " + str(usage.file_position) + ")" + + def is_builtin(self, name: str) -> bool: + return get_built_in_function(name) is not None + + def append_builtin_usage(self, usage: Usage): + bif = get_built_in_function(usage.name) + if bif is not None: + bif.usages.append(usage) + + def get_declaration(self, node: (ast.Name, ast.Attribute, ast.Call, ast.Subscript)) -> Optional[Set[Declaration]]: + if isinstance(node, ast.Name): + declaration = self.scope_manager.get_declaration(node.id) + if declaration is None: + return None + else: + return {declaration} + + mac = MemberAccessCollector(node, self.persistence) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and len(mac.call_list) == 1: + declaration = self.scope_manager.get_declaration(node.func.id) + if declaration is None or isinstance(declaration, PlaceholderType): + bif = get_built_in_function(node.func.id) + if bif is None: + return None + else: + return {bif} + else: + return {declaration} + elif isinstance(node, ast.Call) and len(mac.call_list) == 1 and not isinstance(node.func, ast.Subscript): + pass # print("NEED FIX: () operator called on lambda, operator or await") + + mac = MemberAccessCollector(node, self.persistence) + last = mac.call_list.pop(0) + types = self.type_deduction.get_member_access_type(mac) + declarations = set() + + for d in types: + if isinstance(d, ModuleVariableDeclaration): + if isinstance(d.imported_module, GlobalScope): + declaration = self.scope_manager.get_declaration_from_scope(last.name, d.imported_module) + if declaration is not None: + declarations.add(declaration) + elif isinstance(d.imported_module, PreprocessedFile): + pass + elif d.imported_module is None: + pass + else: + assert False + elif isinstance(d, (VariableDeclaration, FunctionDeclaration, ClassDeclaration)): + if isinstance(d, (TypeVariableDeclaration, FunctionVariableDeclaration)): + current_types = self.type_deduction.get_current_type({d.reference}) + else: + current_types = self.type_deduction.get_current_type(d.type) + for t in current_types: + if isinstance(t, ClassDeclaration): + if t is self.scope_manager.get_current_class_declaration(): + declaration = self.scope_manager.get_declaration_from_current_class(last.name) + if isinstance(declaration, PlaceholderType): + for cl in reversed(self.preprocessed_file.class_collector.classes): + if not cl.file_position == t.file_position: + continue + for a in reversed(cl.attributes): + if a.name == last.name: + declarations.add(a) + for m in reversed(cl.methods): + if m.name == last.name: + if isinstance(last, MemberAccessCollector.AttributeData): + declarations.add(MethodVariableDeclaration(m, t)) + else: + declarations.add(m) + else: + if isinstance(declaration, FunctionDeclaration) and \ + isinstance(last, MemberAccessCollector.AttributeData): + declarations.add(MethodVariableDeclaration(declaration, t)) + else: + declarations.add(declaration) + elif isinstance(last, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for a in t.attributes: + if a.name == last.name: + declarations.add(a) + for a in t.static_attributes: + if a.name == last.name: + declarations.add(a) + elif isinstance(last, MemberAccessCollector.MethodData): + for m in t.methods: + if m.name == last.name: + declarations.add(m) + for m in t.static_methods: + if m.name == last.name: + declarations.add(m) + else: + pass # print("NEED FIX: BuiltIn, InitVariablePlaceholderType, VariablePlaceholderType") + elif isinstance(d, (PlaceholderType, BuiltIn)): + pass + else: + assert False + + return declarations + + # TODO: change PlaceholderType-s, to actual type + # TODO: declaration is None -> what if not declared in global scope (eg. class member) - pass preprocessed 'scope'? + def post_process(self) -> None: + def post_process_variable(var: PreprocessedVariable, class_declaration: Optional[ClassDeclaration]) -> None: + var_declaration = None + if class_declaration is None: + var_declaration = self.scope_manager.get_global_scope().get_variable_by_name(var.name) + else: + for a in reversed(class_declaration.attributes): + if a.name == var.name: + var_declaration = a + break + for a in reversed(class_declaration.static_attributes): + if a.name == var.name: + var_declaration = a + break + if var_declaration is None: + return + var_declaration.usages.extend(var.usages) + for type_usage in var.type_usages: + type_usage.type.update(var_declaration.type) + + def post_process_function(func: PreprocessedFunction, class_declaration: Optional[ClassDeclaration]) -> None: + func_declaration = None + if class_declaration is None: + func_declaration = self.scope_manager.get_global_scope().get_function_by_name(func.name) + else: + for m in reversed(class_declaration.methods): + if m.name == func.name: + func_declaration = m + break + for m in reversed(class_declaration.static_methods): + if m.name == func.name: + func_declaration = m + break + if func_declaration is None: + return + func_declaration.usages.extend(func.usages) + for type_usage in func.type_usages: + type_usage.type.update(func_declaration.type) + + def post_process_class(pcl: PreprocessedClass, class_declaration: Optional[ClassDeclaration]) -> None: + current_class_declaration = None + if class_declaration is None: + current_class_declaration = self.scope_manager.get_global_scope().get_class_by_name(pcl.name) + else: + for cl in reversed(class_declaration.classes): + if c.name == cl.name: + current_class_declaration = c + break + + if current_class_declaration is None: + return + + for cv in pcl.attributes: + post_process_variable(cv, current_class_declaration) + for cf in pcl.methods: + post_process_function(cf, current_class_declaration) + for cc in pcl.classes: + post_process_class(cc, current_class_declaration) + + for type_usage in pcl.type_usages: + type_usage.type.add(current_class_declaration) + + for v in self.preprocessed_file.preprocessed_variables: + post_process_variable(v, None) + for f in self.preprocessed_file.preprocessed_functions: + post_process_function(f, None) + for c in self.preprocessed_file.class_collector.classes: + post_process_class(c, None) + + def create_var_usage(self, name: str, param: ast.expr) -> Usage: + r = create_range_from_ast_node(param) + return Usage(name, FilePosition(self.current_file, r)) + + def create_function_usage(self, func: ast.Call) -> Usage: + name = '' + h = ['func'] + while True: + if eval('hasattr(' + '.'.join(h) + ', "value")'): + if eval('isinstance(' + ".".join(h) + ', str)'): + break + else: + h.append('value') + elif eval('hasattr(' + '.'.join(h) + ', "func")'): + h.append('func') + else: + break + + n = [] + for i in range(len(h), 1, -1): + if eval('hasattr(' + '.'.join(h[:i]) + ', "id")'): + id_attr = "id" + elif eval('hasattr(' + '.'.join(h[:i]) + ', "attr")'): + id_attr = "attr" + else: + continue + n.append(eval("getattr(" + '.'.join(h[:i]) + ", '" + id_attr + "')")) + + # asdf = '.'.join(n) + + mac = MemberAccessCollector(func, self.persistence) + + if hasattr(func.func, 'id'): + name = func.func.id + elif hasattr(func.func, 'attr'): + name = func.func.attr + else: + # print("NEED FIX: usage when () operator called on return value") + assert True # call on return value (eg. f()(), g[i]()) + + r = create_range_from_ast_node(func) + return Usage(name, FilePosition(self.current_file, r)) + + @staticmethod + def is_static_method(node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> bool: + for decorator in node.decorator_list: + if isinstance(decorator, ast.Name) and decorator.id == 'staticmethod': + return True + return False + + def is_static_variable(self, node: ast.AST) -> bool: + return isinstance(self.scope_manager.get_current_scope(), ClassScope) and isinstance(node, ast.Name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py new file mode 100644 index 000000000..8e816e2ee --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from typing import Dict + +from cc_python_parser.base_data import ImportedDeclaration +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.member_access_collector import MemberAccessCollector +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.scope_manager import ScopeManager + + +class ISymbolCollector(ABC): + @abstractmethod + def collect_symbols(self): + pass + + +class SymbolCollectorBase(ISymbolCollector, ABC): + def __init__(self, preprocessed_file: PreprocessedFile, import_finder: ImportFinder): + self.preprocessed_file: PreprocessedFile = preprocessed_file + self.import_finder: ImportFinder = import_finder + self.member_access_collector_type = MemberAccessCollector + self.imported_declaration_scope_map: Dict[ImportedDeclaration, ScopeManager] = {} + + +class IFunctionSymbolCollector(ISymbolCollector, ABC): + def __init__(self): + self.function = None diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py new file mode 100644 index 000000000..c140bf10c --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py @@ -0,0 +1,10 @@ +from typing import Optional +from abc import ABC, abstractmethod + +from cc_python_parser.base_data import Declaration + + +class SymbolFinder(ABC): + @abstractmethod + def get_declaration(self, name: str) -> Optional[Declaration]: + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_data.py b/plugins/python/parser/src/scripts/cc_python_parser/type_data.py new file mode 100644 index 000000000..7e2956d26 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_data.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod + + +class DeclarationType(ABC): + def __init__(self, name: str, qualified_name: str): + self.name = name + self.qualified_name = qualified_name + + @abstractmethod + def get_type_repr(self) -> str: + pass + + @abstractmethod + def __hash__(self): + pass + + @abstractmethod + def __eq__(self, other): + pass + + @abstractmethod + def __ne__(self, other): + pass + + +class PlaceholderType(DeclarationType): + def __init__(self, name: str): + DeclarationType.__init__(self, name, "") + + def get_type_repr(self) -> str: + return 'Placeholder' + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + +class VariablePlaceholderType(PlaceholderType): + pass + + +class InitVariablePlaceholderType(VariablePlaceholderType): + pass + + +class FunctionPlaceholderType(PlaceholderType): + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py b/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py new file mode 100644 index 000000000..85fc7912e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py @@ -0,0 +1,613 @@ +import ast +import typing +from functools import singledispatchmethod +from typing import Optional, Any, Union + +from cc_python_parser import built_in_types +from cc_python_parser.base_data import Declaration, ImportedDeclaration, Usage +from cc_python_parser.built_in_functions import get_built_in_function +from cc_python_parser.built_in_operators import get_built_in_operator +from cc_python_parser.built_in_types import Boolean, Dictionary, Set, Tuple, String, Integer, Float, Bytes, \ + EllipsisType, NoneType, Complex, RangeType, BuiltIn, Type, GenericBuiltInType, NotImplementedType, \ + get_built_in_type, GenericType, Generator +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.hashable_list import OrderedHashableList +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from cc_python_parser.member_access_collector import MemberAccessCollector, MemberAccessCollectorIterator +from cc_python_parser.preprocessed_data import PreprocessedDeclaration +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.scope_manager import ScopeManager +from cc_python_parser.symbol_collector_interface import SymbolCollectorBase +from cc_python_parser.type_data import DeclarationType, PlaceholderType, VariablePlaceholderType, FunctionPlaceholderType, \ + InitVariablePlaceholderType +from cc_python_parser.type_hint_data import TypeHintType +from cc_python_parser.variable_data import VariableDeclaration, FunctionVariableDeclaration, TypeVariableDeclaration, \ + MethodVariableDeclaration + + +class TypeDeduction: + def __init__(self, symbol_collector: SymbolCollectorBase, scope_manager: ScopeManager, + preprocessed_file: PreprocessedFile, + function_symbol_collector_factory: FunctionSymbolCollectorFactory, + persistence: ModelPersistence): + self.scope_manager: ScopeManager = scope_manager + self.preprocessed_file: PreprocessedFile = preprocessed_file + self.symbol_collector: SymbolCollectorBase = symbol_collector + self.function_symbol_collector_factory: FunctionSymbolCollectorFactory = function_symbol_collector_factory + self.container_recursion_counter: int = 0 + self.container_max_recursion: int = 5 + self.persistence: ModelPersistence = persistence + + def deduct_type(self, node: ast.AST) -> typing.Set: + types = self.get_type(node) + if not isinstance(types, typing.Set): + return {types} + return types + + def get_current_type(self, types: typing.Set[DeclarationType]) \ + -> typing.Set[Union[ + ClassDeclaration, BuiltIn, FunctionVariableDeclaration, TypeVariableDeclaration, PlaceholderType]]: + def get_current_type_impl(declaration_type: DeclarationType) \ + -> typing.Set[Union[ + ClassDeclaration, BuiltIn, FunctionVariableDeclaration, TypeVariableDeclaration, PlaceholderType]]: + if isinstance(declaration_type, PlaceholderType): + if isinstance(declaration_type, InitVariablePlaceholderType): + return {declaration_type} + declaration = self.scope_manager.get_declaration(declaration_type.name) + if isinstance(declaration, PlaceholderType) and declaration_type.name == declaration.name: + return {declaration} + if declaration in types: + return set() + tt = {d for d in declaration.type if d not in types} + return self.get_current_type(tt) + elif isinstance(declaration_type, FunctionDeclaration): + return self.get_current_type(declaration_type.type) + elif isinstance(declaration_type, + (ClassDeclaration, BuiltIn, FunctionVariableDeclaration, TypeVariableDeclaration)): + return {declaration_type} + elif isinstance(declaration_type, VariableDeclaration): + return self.get_current_type(declaration_type.type) + elif isinstance(declaration_type, TypeHintType): + return {declaration_type.hinted_type} + elif isinstance(declaration_type, PreprocessedDeclaration): + return {declaration_type} + else: + assert False + + current_types = set() + for t in types: + if t is None: + continue + current_types.update(get_current_type_impl(t)) + return current_types + + @singledispatchmethod + def get_type(self, node: ast.AST) -> Any: + assert False, "Unknown type: " + str(type(node)) + + @get_type.register + def _(self, node: ast.BoolOp): + return Boolean() + + @get_type.register + def _(self, node: ast.NamedExpr): # (x := y) eg. if (y := f(x)) is not None: ... + pass + + @get_type.register + def _(self, node: ast.BinOp): + left_operand_type = self.get_current_type(self.deduct_type(node.left)) + bio = get_built_in_operator(node.op) + assert bio is not None + types = set() + for t in left_operand_type: + if isinstance(t, (BuiltIn, PlaceholderType)): + types.add(t) # TODO: handle numbers + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: check this (eg. base class is BuiltIn) + types.add(t) + else: + pass # print("???") + return types # type(left_operand_type.op) + + @get_type.register + def _(self, node: ast.UnaryOp): + operand = self.get_current_type(self.deduct_type(node.operand)) + bio = get_built_in_operator(node.op) + assert bio is not None + types = set() + for t in operand: + if isinstance(t, BuiltIn): + types.add(t) + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: preprocessed method + types.add(t) + else: + pass # print("???") + return types + + @get_type.register + def _(self, node: ast.Lambda): + # add node.args to tmp_scope + return self.get_type(node.body) + + @get_type.register + def _(self, node: ast.IfExp): + types = self.deduct_type(node.body) + types.update(self.deduct_type(node.orelse)) + return types + + @get_type.register + def _(self, node: ast.Dict): + return Dictionary(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.Set): + return Set(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.ListComp): + return built_in_types.List() # generic types? + + @get_type.register + def _(self, node: ast.SetComp): + return Set() # generic types? + + @get_type.register + def _(self, node: ast.DictComp): + return Dictionary() # generic types? + + @get_type.register + def _(self, node: ast.GeneratorExp): + g = Generator() + for generator in node.generators: + g.types.update(self.deduct_type(generator.iter)) + return g # tmp_var := type(iter) -> type(target) + + @get_type.register + def _(self, node: ast.Await): + return self.get_type(node.value) + + @get_type.register + def _(self, node: ast.Compare): + # TODO: handle PlaceHolderType + left_values = self.deduct_type(node.left) + types = set() + assert len(node.comparators) == len(node.ops) + for i in range(0, len(node.comparators)): + bio = get_built_in_operator(node.ops[i]) + assert bio is not None + left_operand_type = self.get_current_type(left_values) + for t in left_operand_type: + if isinstance(t, BuiltIn): + types.add(Boolean()) + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: check this (eg. base class is BuiltIn) + types.add(t) + else: + pass # print("???") + left_values = types + return left_values + + @get_type.register + def _(self, node: ast.Call): + if hasattr(node.func, 'id'): + # TODO: check if builtin class ctor-s are not hidden by other (custom) functions/classes + # TODO: no args check (builtin is not hidden + correct code -> should not be a problem) + built_in_function = get_built_in_function(node.func.id) + if built_in_function is not None: + return built_in_function + elif node.func.id == 'TypeVar': + return Type() + return self.get_member_access_type(MemberAccessCollector(node, self.persistence), + FilePosition(self.scope_manager.current_file, + create_range_from_ast_node(node))) + + @get_type.register + def _(self, node: ast.FormattedValue): # {x} in a joined string + return self.get_type(node.value) # cannot be on right side without f-string? + + @get_type.register + def _(self, node: ast.JoinedStr): # f"... {x} ... {y} ..." + return String() + + @get_type.register + def _(self, node: ast.Constant): + if isinstance(node.value, int): + if isinstance(node.value, bool): + return Boolean() + else: + return Integer() + elif isinstance(node.value, float): + return Float() + elif isinstance(node.value, bytes): + return Bytes() + elif isinstance(node.value, complex): + return Complex() + elif isinstance(node.value, str): + return String() + elif node.value is None: + return NoneType() + elif node.value is Ellipsis: + return EllipsisType() + return set() + + @get_type.register + def _(self, node: ast.Attribute): + return self.get_member_access_type(MemberAccessCollector(node, self.persistence), + FilePosition(self.scope_manager.current_file, + create_range_from_ast_node(node))) + + @get_type.register + def _(self, node: ast.Subscript): + if isinstance(node.slice, ast.Index): + if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) + return Integer() + else: + if isinstance(node.value, (ast.Name, ast.Attribute, ast.Subscript, ast.Call, ast.Tuple, + ast.ListComp, ast.List)): + t = self.deduct_type(node.value) + elif isinstance(node.value, (ast.Dict, ast.DictComp)): + t = set() + elif isinstance(node.value, ast.Constant): + t = {String()} # TODO: can be other than str? + elif isinstance(node.value, (ast.BinOp, ast.UnaryOp)): + t = set() + operand_types = self.deduct_type(node.value) + bio = get_built_in_operator(node.value.op) + assert bio is not None + for operand_type in self.get_current_type(operand_types): + if isinstance(operand_type, ClassDeclaration): + override = [x for x in operand_type.methods if x.name in bio.get_override()] + if len(override) > 0: + t.add(override[0]) + else: + assert False, f"Unhandled type: {type(node.value)}" + generic_types = set() + subscript_type_sets = [a.type for a in t if not isinstance(a, PlaceholderType)] + for subscript_types in subscript_type_sets: + for subscript_type in subscript_types: + if isinstance(subscript_type, GenericType): + for generic_type in subscript_type.types: + if isinstance(generic_type, ImportedDeclaration): + generic_types.add(generic_type.imported_declaration) + else: + generic_types.add(generic_type) + return generic_types + elif isinstance(node.slice, (ast.Slice, ast.ExtSlice)): # ExtSlice -> array type (module arr) + if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) + return RangeType() + else: + return set() # type(node.value)[type(node.value->GenericType)] + return set() # {type(value)...} + + @get_type.register + def _(self, node: ast.Starred): # unpack: * (eg. list, tuple) and ** (eg. dictionary) + return set() # could be same as iterator + + @get_type.register + def _(self, node: ast.Name): + bit = get_built_in_type(node.id) + if bit is not None: + return bit + if node.id == 'NotImplemented': + return NotImplementedType() + elif node.id == 'Ellipsis': + return EllipsisType() + return self.scope_manager.get_variable_declaration(node.id) + + @get_type.register + def _(self, node: ast.List): + return built_in_types.List(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.Tuple): + return Tuple(self.get_element_types(node)) + + @get_type.register(ast.Return) + @get_type.register(ast.Yield) + @get_type.register(ast.YieldFrom) + def _(self, node): + return self.get_type_of_function(node) + + @get_type.register + def _(self, node: ast.Expr): + return self.get_type(node.value) + + def get_element_types(self, node: ast.AST): + # TODO: need better soultion eg: Lib > test > test_parser.py (tuple) + if self.container_recursion_counter < self.container_recursion_counter: + return set() + self.container_recursion_counter += 1 + element_types = set() + if isinstance(node, ast.Dict): + elements = getattr(node, 'values') + else: + assert hasattr(node, 'elts') + elements = getattr(node, 'elts') + for elem in elements: + elem_type = self.get_type(elem) + if elem_type is not None: + if isinstance(elem_type, typing.Set): + element_types.update(elem_type) + else: + element_types.add(elem_type) + self.container_recursion_counter -= 1 + return element_types + + def get_member_access_type(self, mac: MemberAccessCollector, pos: Optional[FilePosition] = None): + if len(mac.call_list) == 0: + return set() + iterator = MemberAccessCollectorIterator(mac) + declaration = self.scope_manager.get_declaration_from_member_access(iterator) + if (declaration is None or isinstance(declaration, PlaceholderType)) and pos is not None: + if iterator.is_iteration_started(): + a = iterator.get_current() + else: + a = iterator.get_first() + + # TODO: is empty FilePosition correct? + if isinstance(a, MemberAccessCollector.MethodData): + b = [x for x in self.preprocessed_file.preprocessed_functions if x.name == a.name] + c = [x for x in self.preprocessed_file.class_collector.classes if x.name == a.name] + for i in b: + i.usages.append(Usage(a.name, pos)) + for i in c: + i.usages.append(Usage(a.name, pos)) + elif isinstance(a, (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + b = [x for x in self.preprocessed_file.preprocessed_variables if x.name == a.name] + for i in b: + i.usages.append(Usage(a.name, pos)) + declarations = {declaration} + if iterator.is_iteration_over() or not iterator.is_iteration_started(): + if not any(isinstance(x, PlaceholderType) for x in declarations) and \ + any(any(isinstance(x, PlaceholderType) for x in y.type) for y in declarations): + if any(isinstance(x, FunctionVariableDeclaration) for x in declarations): + pass # print() + else: + declarations = self.get_current_type_of_placeholder_function(declarations, mac.call_list[-1]) + return declarations + prev_member = iterator.get_current() + next(iterator) + for member in mac.call_list[iterator.index::-1]: + if len(declarations) == 0 or all(x is None for x in declarations) or \ + all(isinstance(x, PlaceholderType) for x in declarations): + return set() + declarations = self.get_member_declarations(declarations, prev_member, member) + prev_member = member + return declarations + + def get_member_declarations(self, declarations: typing.Set[DeclarationType], + current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + member_declarations = set() + for declaration in declarations: + for declaration_type in self.get_current_type(declaration.type): + if isinstance(declaration_type, PlaceholderType) and isinstance(declaration, FunctionDeclaration): + d = self.get_current_type_of_placeholder_function({declaration}, current_member) + for dt in d: + for declaration_current_type in self.get_possible_types(dt): + if isinstance(declaration_current_type, BuiltIn): + continue + elif isinstance(declaration_current_type, ClassDeclaration): + member_declarations.update( + self.get_member(declaration_current_type, current_member, next_member)) + elif isinstance(declaration_current_type, (PlaceholderType, VariablePlaceholderType)): + continue # TODO: handle this (eg. function parameters) + else: + assert False + # member_declarations.update(d) + continue # TODO: try to get current value of function with the arguments from 'mac' + for declaration_current_type in self.get_possible_types(declaration_type): + if isinstance(declaration_current_type, BuiltIn): + self.get_member_of_builtin(declaration_current_type, current_member, next_member) + elif isinstance(declaration_current_type, ClassDeclaration): + member_declarations.update(self.get_member( + declaration_current_type, current_member, next_member)) + elif isinstance(declaration_current_type, VariablePlaceholderType): + continue # TODO: handle this (eg. function parameters) + elif isinstance(declaration_current_type, InitVariablePlaceholderType): + continue + elif isinstance(declaration_current_type, PlaceholderType): + continue + else: + self.get_possible_types(declaration_type) + assert False, f"Unhandled type: {type(declaration_current_type)}" + return member_declarations + + def get_member(self, declaration: Declaration, current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + types = self.get_possible_types(declaration) + declarations = set() + pf = None + for t in types: + if isinstance(t, ClassDeclaration): + if t is self.scope_manager.get_current_class_declaration(): + pf = self.preprocessed_file + pc = None + for c in self.preprocessed_file.class_collector.classes: + if c.file_position == t.file_position: + if isinstance(next_member, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for attr in reversed(c.attributes): + if attr.name == next_member.name: + pass + elif isinstance(next_member, MemberAccessCollector.MethodData): + for method in reversed(c.methods): + if method.name == next_member.name: + arguments = self.collect_current_argument_types(next_member) + # t = self.evaluate_function_def(method.node, call_on, arguments) + break + if pc is not None: + pass + # else: + if isinstance(next_member, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for attr in reversed(t.attributes): + if attr.name == next_member.name: + declarations.add(attr) + for attr in reversed(t.static_attributes): + if attr.name == next_member.name: + declarations.add(attr) + elif isinstance(next_member, MemberAccessCollector.MethodData): + for method in reversed(t.methods): + if method.name == next_member.name: + declarations.add(method) + for method in reversed(t.static_methods): + if method.name == next_member.name: + declarations.add(method) + elif isinstance(next_member, MemberAccessCollector.SubscriptData): + # TODO: subscript elements type -> init/call method + pass + elif isinstance(next_member, MemberAccessCollector.LambdaData): + # TODO: member.node type -> init/call method + pass + elif isinstance(next_member, MemberAccessCollector.OperatorData): + # TODO: member.node type -> init/call method + pass + elif isinstance(t, BuiltIn): + continue + else: + assert False + return declarations + + def get_member_of_builtin(self, builtin: BuiltIn, current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + if isinstance(builtin, GenericBuiltInType) and isinstance(current_member, MemberAccessCollector.SubscriptData): + return builtin.types + return set() + + # evaluate declaration to get type + def get_possible_types(self, declaration: DeclarationType) -> typing.Set[Union[ClassDeclaration, BuiltIn]]: + types = set() + # TODO: handle BuiltIn-s (Module!) + if isinstance(declaration, BuiltIn): + return {declaration} + elif isinstance(declaration, Declaration): + for t in declaration.type: + if isinstance(t, (ClassDeclaration, BuiltIn, PlaceholderType)): + types.add(t) + elif isinstance(t, (VariableDeclaration, FunctionDeclaration)): + types.update(self.get_possible_types(t)) + else: + assert False, "Unknown type: " + str(type(t)) + elif isinstance(declaration, PlaceholderType): + # TODO: get current value of function or variable + return self.get_current_type({self.scope_manager.get_declaration(declaration.name)}) + else: + assert False, "Unknown declaration type: " + str(type(declaration)) + return types + + def get_current_type_of_placeholder_function(self, declarations: typing.Set, + member: MemberAccessCollector.MemberData) \ + -> typing.Set[DeclarationType]: + types = set() + + for declaration in declarations: + if isinstance(declaration, ImportedDeclaration): + if declaration not in self.symbol_collector.imported_declaration_scope_map: + continue # TODO: fix this, eg: Lib > unittest > test > test_discovery.py > in def restore_listdir + scope_manager = self.symbol_collector.imported_declaration_scope_map[declaration] + func_def = scope_manager.placeholder_function_declaration_cache. \ + get_function_def(declaration.imported_declaration) + elif isinstance(declaration, FunctionVariableDeclaration): + func_def = self.scope_manager.placeholder_function_declaration_cache. \ + get_function_def(declaration.reference) + else: + func_def = self.scope_manager.placeholder_function_declaration_cache.get_function_def(declaration) + # assert func_def is not None # VariableDeclaration? + if func_def is None: + continue + + arguments = OrderedHashableList() + if isinstance(declaration, MethodVariableDeclaration): + arguments.append((declaration.self, None)) + elif func_def.is_method: + arguments.append((self.get_current_type({declaration}), None)) + arguments.extend(self.collect_current_argument_types(member)) + declaration_type = self.scope_manager.placeholder_function_declaration_cache. \ + get_functions_return_type(declarations, arguments) + if declaration_type is not None and len(declaration_type) > 0: + types.update(declaration_type) + else: + types.update(self.evaluate_function_def(func_def.function, arguments)) + return types + + def collect_current_argument_types(self, method_data: MemberAccessCollector.MemberData) -> OrderedHashableList: + arguments = OrderedHashableList() + if isinstance(method_data, MemberAccessCollector.MethodData): + for arg in method_data.arguments: + arguments.append((self.deduct_type(arg), None)) + + for kw in method_data.keywords: + arguments.append((self.deduct_type(kw.argument), kw.name)) + return arguments + + def evaluate_function_def(self, function_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: OrderedHashableList) -> typing.Set[DeclarationType]: + types = set() + if len(arguments) == 0: + sc = self.function_symbol_collector_factory.get_function_symbol_collector( + self.symbol_collector, function_def, []) + sc.collect_symbols() + types.update(sc.function.type) + else: + for args in self.get_argument_combinations(arguments): + sc = self.function_symbol_collector_factory.get_function_symbol_collector( + self.symbol_collector, function_def, args) + sc.collect_symbols() + types.update(sc.function.type) + return types + + def get_argument_combinations(self, arguments: + OrderedHashableList[typing.Tuple[typing.Set[DeclarationType], Optional[str]]]) \ + -> typing.Iterator[OrderedHashableList[typing.Tuple[DeclarationType, Optional[str]]]]: + if len(arguments) == 0: + return OrderedHashableList() + for argument in arguments[0][0]: + head_argument = OrderedHashableList([(argument, arguments[0][1])]) + if len(arguments) == 1: + yield head_argument + else: + for tail_arguments in self.get_argument_combinations(arguments[1::]): + yield head_argument + tail_arguments + + def get_type_of_function(self, node: Union[ast.Return, ast.Yield, ast.YieldFrom]): + if node.value is None: + return set() + + types = self.get_type(node.value) + if types is None: + return set() + fixed_types = set() + if isinstance(types, typing.Set): + fixed_types.update(map(lambda x: self.fix_placeholder(x), types)) + else: + fixed_types.add(self.fix_placeholder(types)) + return fixed_types + + def fix_placeholder(self, declaration_type: DeclarationType): + if any(isinstance(x, PlaceholderType) for x in self.get_current_type({declaration_type})): + if isinstance(declaration_type, VariableDeclaration): + return VariablePlaceholderType(declaration_type.name) + elif isinstance(declaration_type, FunctionDeclaration): + return FunctionPlaceholderType(declaration_type.name) + return declaration_type diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py b/plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py new file mode 100644 index 000000000..f54e7cff6 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py @@ -0,0 +1,23 @@ +from typing import Union + +from cc_python_parser.built_in_types import BuiltIn +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.type_data import DeclarationType + + +class TypeHintType(DeclarationType): + def __init__(self, hinted_type: Union[ClassDeclaration, BuiltIn]): + super().__init__(hinted_type.name, hinted_type.qualified_name) + self.hinted_type = hinted_type + + def get_type_repr(self) -> str: + return 'Hinted type: ' + self.hinted_type.get_type_repr() + + def __hash__(self): + return hash(self.hinted_type) + + def __eq__(self, other): + return isinstance(other, TypeHintType) and self.hinted_type == other.hinted_type + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py b/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py new file mode 100644 index 000000000..5764f088c --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py @@ -0,0 +1,101 @@ +from pathlib import PurePath +from typing import Optional, Set + +from cc_python_parser.built_in_types import Module, AnyType, Method, Function, Type +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.base_data import Declaration, TypeDeclaration, ImportedDeclaration, ReferenceDeclaration +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.persistence.base_dto import UsageDTO +from cc_python_parser.persistence.variable_dto import VariableDeclarationDTO +from cc_python_parser.type_data import PlaceholderType + + +class VariableDeclaration(Declaration): + # var_type: Optional[Set[Union[cd.ClassDeclaration, BuiltIn]]] - circular import + def __init__(self, name: str, qualified_name: str, pos: FilePosition, declaration_type: Optional[Set] = None): + super().__init__(name, qualified_name, pos, declaration_type) + # self.type: VariableDeclaration and FunctionDeclaration type can change + # (eg. new assignment with different type or redefinition) + + def create_dto(self) -> VariableDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + for t in self.type: + if isinstance(t, PlaceholderType): + types.add(AnyType().qualified_name) + elif isinstance(t, MethodVariableDeclaration): + types.add(Method().qualified_name) + elif isinstance(t, FunctionVariableDeclaration): + types.add(Function().qualified_name) + elif isinstance(t, TypeVariableDeclaration): + types.add(Type().qualified_name) + elif isinstance(t, ImportedDeclaration): + types.add(t.imported_declaration.qualified_name) + elif isinstance(t, ReferenceDeclaration): + types.add(t.reference.qualified_name) + else: + types.add(t.qualified_name) + return VariableDeclarationDTO(self.name, self.qualified_name, self.file_position, types, usages) + + +class StaticVariableDeclaration(VariableDeclaration): + pass + + +class ReferenceVariableDeclaration(VariableDeclaration, ReferenceDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: Declaration): + VariableDeclaration.__init__(self, name, qualified_name, pos, reference.type) + ReferenceDeclaration.__init__(self, reference) + + +class NonlocalVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): + super().__init__(name, qualified_name, pos, reference) + + +class GlobalVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): + super().__init__(name, qualified_name, pos, reference) + + +# TODO: short/long name? (in case of import) +class ModuleVariableDeclaration(VariableDeclaration): + # module: Union[GlobalScope, PreprocessedFile] - circular import + def __init__(self, qualified_name: str, name: str, location: PurePath, pos: FilePosition, module): + super().__init__(name, qualified_name, pos, {Module()}) + self.imported_module = module + self.imported_module_location: PurePath = location + + +class ImportedVariableDeclaration(VariableDeclaration, ImportedDeclaration[VariableDeclaration]): + def __init__(self, qualified_name: str, name: str, pos: FilePosition, var_declaration: VariableDeclaration, module): + VariableDeclaration.__init__(self, name, "", pos, var_declaration.type) + ImportedDeclaration.__init__(self, qualified_name, var_declaration, module, pos) + + +class TypeVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, type_ref: TypeDeclaration): + super().__init__(type_ref.name, type_ref.qualified_name, type_ref.file_position, type_ref) + + def get_type_repr(self) -> str: + return '[TypeVariable(' + self.reference.name + ')]' + + +class FunctionVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, func_ref: FunctionDeclaration): + super().__init__(func_ref.name, func_ref.qualified_name, func_ref.file_position, func_ref) + + def get_type_repr(self) -> str: + return '[FunctionVariable(' + self.reference.name + ')]' + + +class MethodVariableDeclaration(FunctionVariableDeclaration): + # self_var: ClassDeclaration + def __init__(self, func_ref: FunctionDeclaration, self_var): + super().__init__(func_ref) + self.self = self_var + + def get_type_repr(self) -> str: + return '[MethodVariable(' + self.reference.name + ')]' diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt new file mode 100644 index 000000000..4ddc22a15 --- /dev/null +++ b/plugins/python/service/CMakeLists.txt @@ -0,0 +1,29 @@ +include_directories( + include + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp + ${PROJECT_SOURCE_DIR}/service/project/include + ${PLUGIN_DIR}/model/include) + +include_directories(SYSTEM + ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) + +add_library(pythonservice SHARED + src/pythonservice.cpp + src/plugin.cpp) + +target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) + +target_link_libraries(pythonservice + util + model + pythonmodel + mongoose + projectservice + languagethrift + gvc + ${THRIFT_LIBTHRIFT_LIBRARIES}) + +install(TARGETS pythonservice DESTINATION ${INSTALL_SERVICE_DIR}) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h new file mode 100644 index 000000000..3d59fe8f3 --- /dev/null +++ b/plugins/python/service/include/service/pythonservice.h @@ -0,0 +1,239 @@ +#ifndef CC_SERVICE_LANGUAGE_PYTHONSERVICE_H +#define CC_SERVICE_LANGUAGE_PYTHONSERVICE_H + +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +class PythonServiceHandler : virtual public LanguageServiceIf +{ +public: + PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + void getFileTypes(std::vector& return_) override; + + void getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) override; + + void getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) override; + + void getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) override; + + void getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) override; + + void getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) override; + + void getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) override; + + std::int32_t getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) override; + + void getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) override; + + void getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) override; + + void getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + std::int32_t getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + void getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) override; + +private: + enum ReferenceType + { + DECLARATION, /*!< By this options the declaration(s) of the AST node can be + queried. */ + + USAGE, /*!< By this option the usages of the AST node can be queried, i.e. + the nodes of which the mangled name is identical to the queried one. The + parser creates a mangled name even for variables and types, which is + unique for the element. */ + + THIS_CALLS, /*!< Get function calls in a function. WARNING: If the + definition of the AST node is not unique then it returns the callees of + one of them. */ + + CALLS_OF_THIS, /*!< Get calls of a function. */ + + CALLEE, /*!< Get called functions definitions. WARNING: If the definition of + the AST node is not unique then it returns the callees of one of them. */ + + CALLER, /*!< Get caller functions. */ + + PARAMETER, /*!< This option returns the parameters of a function. */ + + LOCAL_VAR, /*!< This option returns the local variables of a function. */ + + RETURN_TYPE, /*!< This option returns the return type of a function. */ + + TYPE, /*!< This option returns the type of a variable. */ + + INHERIT_FROM, /*!< Types from which the queried type inherits. */ + + INHERIT_BY, /*!< Types by which the queried type is inherited. */ + + DATA_MEMBER, /*!< Data members of a class. */ + + METHOD, /*!< Members of a class. */ + + NESTED_CLASS, /*!< Nested classes. */ + + IMPORTED_SYMBOLS + }; + + enum FileReferenceType + { + IMPORTS, /*!< Imported modules in the current module. */ + + CLASSES, /*!< User defined classes */ + + FUNCTIONS, /*!< Functions in the current module. */ + + VARIABLES /*!< Variables in the current module. */ + }; + + static bool compareByPosition( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs); + + static bool compareByValue( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs); + + model::PythonAstNode queryPythonAstNode(const core::AstNodeId& astNodeId_); + + std::vector queryPythonAstNodes( + const core::AstNodeId& astNodeId_, + const odb::query& query_ + = odb::query(true)); + + std::vector queryPythonAstNodesInFile( + const core::FileId& fileId_, + const odb::query& query_ + = odb::query(true)); + + std::uint32_t queryPythonAstNodeCountInFile( + const core::FileId& fileId_, + const odb::query& query_ + = odb::query(true)); + + std::vector queryDeclarations(const core::AstNodeId& astNodeId_); + + odb::query astCallsQuery(const model::PythonAstNode& astNode_); + + std::vector queryCalls(const core::AstNodeId& astNodeId_); + + std::size_t queryPythonAstNodeCount( + const core::AstNodeId& astNodeId_, + const odb::query& query_ + = odb::query(true)); + + std::size_t queryCallsCount(const core::AstNodeId& astNodeId_); + + std::vector queryTypes(const model::PythonEntity& entity); + + model::PythonEntity queryPythonEntity(const model::PythonEntityId& id); + + model::PythonEntity queryPythonEntityByAstNode(const std::string& qualifiedName); + + std::map getVisibilities(const std::vector& nodes_); + + bool isGeneratedVariable(const model::PythonAstNode& node_) const; + + std::shared_ptr _db; + util::OdbTransaction _transaction; + + std::shared_ptr _datadir; + const cc::webserver::ServerContext& _context; +}; + +} +} +} + +#endif diff --git a/plugins/python/service/src/plugin.cpp b/plugins/python/service/src/plugin.cpp new file mode 100644 index 000000000..016babd12 --- /dev/null +++ b/plugins/python/service/src/plugin.cpp @@ -0,0 +1,26 @@ +#include + +#include + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + boost::program_options::options_description description("Python Plugin"); + return description; + } + + void registerPlugin( + const cc::webserver::ServerContext& context_, + cc::webserver::PluginHandler* pluginHandler_) + { + cc::webserver::registerPluginSimple( + context_, + pluginHandler_, + CODECOMPASS_LANGUAGE_SERVICE_FACTORY_WITH_CFG(Python), + "PythonService"); + } +} +#pragma clang diagnostic pop diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp new file mode 100644 index 000000000..724f063a7 --- /dev/null +++ b/plugins/python/service/src/pythonservice.cpp @@ -0,0 +1,1097 @@ +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + typedef odb::query AstQuery; + typedef odb::result AstResult; + typedef odb::query EntityQuery; + typedef odb::result EntityResult; + typedef odb::query VarQuery; + typedef odb::result VarResult; + typedef odb::query FuncQuery; + typedef odb::result FuncResult; + typedef odb::query ClassQuery; + typedef odb::result ClassResult; + typedef odb::query ClassMemQuery; + typedef odb::result ClassMemResult; + typedef odb::query ModImpQuery; + typedef odb::result ModImpResult; + typedef odb::query InhQuery; + typedef odb::result InhResult; + typedef odb::query TypeQuery; + typedef odb::result TypeResult; + typedef odb::query DocQuery; + typedef odb::result DocResult; + typedef odb::query FileQuery; + typedef odb::result FileResult; + + struct CreateAstNodeInfo + { + typedef std::map VisibilityMap; + + CreateAstNodeInfo(const VisibilityMap& visibilities_ = {}) : _visibilities(visibilities_) + { + } + + cc::service::language::AstNodeInfo operator()(const cc::model::PythonAstNode& astNode_) + { + cc::service::language::AstNodeInfo ret; + + ret.__set_id(std::to_string(astNode_.id)); + ret.__set_entityHash(cc::model::createAstNodeInfoEntityHash(astNode_)); + ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); + ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); + std::size_t start = astNode_.qualifiedName.find_last_of(".") + 1; + std::size_t size = astNode_.qualifiedName.find_last_of(":") - start; + ret.__set_astNodeValue(astNode_.qualifiedName.substr(start, size)); + + VisibilityMap::const_iterator it = _visibilities.find(astNode_.id); + if (it != _visibilities.end()){ + ret.__set_tags({it->second}); + } + + ret.range.range.startpos.line = astNode_.location.range.start.line; + ret.range.range.startpos.column = astNode_.location.range.start.column+1; + ret.range.range.endpos.line = astNode_.location.range.end.line; + ret.range.range.endpos.column = astNode_.location.range.end.column+1; + + if (astNode_.location.file){ + ret.range.file = std::to_string(astNode_.location.file.object_id()); + } + + return ret; + } + + const VisibilityMap& _visibilities; + }; +} + +namespace cc { +namespace service { +namespace language { + +PythonServiceHandler::PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : _db(db_), + _transaction(db_), + _datadir(datadir_), + _context(context_) +{ +} + +void PythonServiceHandler::getFileTypes(std::vector& return_) +{ + return_.push_back("PY"); + return_.push_back("Dir"); +} + +void PythonServiceHandler::getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) +{ + return_ = _transaction([this, &astNodeId_](){ + return CreateAstNodeInfo()(queryPythonAstNode(astNodeId_)); + }); +} + +void PythonServiceHandler::getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) +{ + _transaction([&, this](){ + //--- Query nodes at the given position ---// + + AstResult nodes = _db->query( + AstQuery::location.file == std::stoull(fpos_.file) && + // StartPos <= Pos + ((AstQuery::location.range.start.line == fpos_.pos.line && + AstQuery::location.range.start.column <= fpos_.pos.column-1) || + AstQuery::location.range.start.line < fpos_.pos.line) && + // Pos < EndPos + ((AstQuery::location.range.end.line == fpos_.pos.line && + AstQuery::location.range.end.column > fpos_.pos.column-1) || + AstQuery::location.range.end.line > fpos_.pos.line)); + + //--- Select innermost clickable node ---// + + model::Range minRange(model::Position(0, 0), model::Position()); + model::PythonAstNode min; + + int size = 0; + for (const model::PythonAstNode& node : nodes) + { + if (!isGeneratedVariable(node) && node.location.range < minRange) + { + min = node; + minRange = node.location.range; + } + } + + return_ = _transaction([this, &min](){ + if(min.astType == model::PythonAstNode::AstType::Declaration){ + return CreateAstNodeInfo(getVisibilities({min}))(min); + } + return CreateAstNodeInfo(getVisibilities({}))(min); + }); + }); +} + +void PythonServiceHandler::getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + return_ = _transaction([this, &astNodeId_](){ + model::PythonAstNode astNode = queryPythonAstNode(astNodeId_); + + if (astNode.location.file){ + return cc::util::textRange( + astNode.location.file.load()->content.load()->content, + astNode.location.range.start.line, + astNode.location.range.start.column, + astNode.location.range.end.line, + astNode.location.range.end.column); + } + + return std::string(); + }); +} + +void PythonServiceHandler::getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + _transaction([&, this](){ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + model::PythonEntity entity = queryPythonEntityByAstNode(node.qualifiedName); + + DocResult doc = _db->query( + DocQuery::documented == entity.id); + + if (doc.empty()){ + return_ = std::string(); + } else { + return_ = doc.begin()->documentation; + } + }); +} + +void PythonServiceHandler::getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + _transaction([&, this](){ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + switch (node.symbolType) + { + case model::PythonAstNode::SymbolType::Variable: + { + VarResult variables = _db->query( + VarQuery::qualifiedName == node.qualifiedName); + model::PythonVariable variable = *variables.begin(); + + return_["Name"] = variable.name; + return_["Qualified name"] = variable.qualifiedName; + break; + } + + case model::PythonAstNode::SymbolType::Function: + { + FuncResult functions = _db->query( + FuncQuery::qualifiedName == node.qualifiedName); + model::PythonFunction function = *functions.begin(); + + return_["Name"] = function.qualifiedName.substr( + function.qualifiedName.find_last_of('.') + 1); + return_["Qualified name"] = function.qualifiedName; + return_["Signature"] = function.name; // TODO + + break; + } + + case model::PythonAstNode::SymbolType::Class: + { + ClassResult classes = _db->query( + ClassQuery::qualifiedName == node.qualifiedName); + model::PythonClass cl = *classes.begin(); + + return_["Name"] = cl.name; + return_["Qualified name"] = cl.qualifiedName; + + break; + } + + case model::PythonAstNode::SymbolType::Module: + { + ModImpResult modules = _db->query( + ModImpQuery::astNodeId == node.id); + model::PythonImport module = *modules.begin(); + return_["From"] = module.imported.load()->filename; + + break; + } + } + }); +} + +void PythonServiceHandler::getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + +} + +void PythonServiceHandler::getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) +{ + +} + +void PythonServiceHandler::getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId); + + return_["Declaration"] = DECLARATION; + return_["Usage"] = USAGE; + + switch (node.symbolType){ + case model::PythonAstNode::SymbolType::Variable: + return_["Type"] = TYPE; + break; + + case model::PythonAstNode::SymbolType::Function: + return_["This calls"] = THIS_CALLS; + return_["Callee"] = CALLEE; + return_["Caller"] = CALLER; + return_["Parameters"] = PARAMETER; + return_["Local variables"] = LOCAL_VAR; + return_["Return type"] = RETURN_TYPE; + break; + + case model::PythonAstNode::SymbolType::Class: + return_["Inherits from"] = INHERIT_FROM; + return_["Inherited by"] = INHERIT_BY; + return_["Data member"] = DATA_MEMBER; + return_["Method"] = METHOD; + return_["Nested class"] = NESTED_CLASS; + break; + + case model::PythonAstNode::SymbolType::Module: + return_["Imported symbols"] = IMPORTED_SYMBOLS; + break; + } +} + +void PythonServiceHandler::getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& /*tags_*/) +{ + std::vector nodes; + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + return _transaction([&, this](){ + switch (referenceId_) + { + case DECLARATION: + nodes = queryDeclarations(astNodeId_); + break; + + case USAGE: + nodes = queryPythonAstNodes(astNodeId_); + break; + + case THIS_CALLS: + nodes = queryCalls(astNodeId_); + break; + + case CALLS_OF_THIS: + nodes = queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Usage); + break; + + case CALLEE: + for (const model::PythonAstNode& call : queryCalls(astNodeId_)) + { + core::AstNodeId astNodeId = std::to_string(call.id); + std::vector defs = queryDeclarations(astNodeId); + nodes.insert(nodes.end(), defs.begin(), defs.end()); + } + + std::sort(nodes.begin(), nodes.end()); + nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); + + break; + + case CALLER: + for (const model::PythonAstNode& astNode : queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Usage)) + { + const model::Position& start = astNode.location.range.start; + const model::Position& end = astNode.location.range.end; + + AstResult result = _db->query( + AstQuery::astType == model::PythonAstNode::AstType::Declaration && + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + // Same file + AstQuery::location.file == astNode.location.file.object_id() && + // StartPos >= Pos + ((AstQuery::location.range.start.line == start.line && + AstQuery::location.range.start.column <= start.column) || + AstQuery::location.range.start.line < start.line) && + // Pos > EndPos + ((AstQuery::location.range.end.line == end.line && + AstQuery::location.range.end.column > end.column) || + AstQuery::location.range.end.line > end.line)); + + nodes.insert(nodes.end(), result.begin(), result.end()); + } + + std::sort(nodes.begin(), nodes.end()); + nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); + + break; + + case PARAMETER: + { + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + for (auto var : function.parameters){ + nodes.push_back(queryPythonAstNode( + std::to_string(var.load()->astNodeId))); + } + + break; + } + + case LOCAL_VAR: + { + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + for (auto var : function.locals){ + nodes.push_back(queryPythonAstNode( + std::to_string(var.load()->astNodeId))); + } + + break; + } + + case RETURN_TYPE: + { + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + std::vector types = queryTypes(function); + + for (const model::PythonClass& cl : types) + { + std::vector defs = + queryDeclarations(std::to_string(cl.astNodeId)); + nodes.insert(nodes.end(), defs.begin(), defs.end()); + } + + break; + } + + case TYPE: + { + VarResult varNodes = _db->query( + VarQuery::qualifiedName == node.qualifiedName); + + const model::PythonVariable& variable = *varNodes.begin(); + + std::vector types = queryTypes(variable); + + for (const model::PythonClass& cl : types) + { + std::vector defs = + queryDeclarations(std::to_string(cl.astNodeId)); + nodes.insert(nodes.end(), defs.begin(), defs.end()); + } + + break; + } + + case INHERIT_FROM: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + + for (const model::PythonInheritance& inh : + _db->query( + InhQuery::derived == cl.id)) + { + model::PythonEntity cl = _db->query_value( + EntityQuery::id == inh.base); + AstResult result = _db->query( + AstQuery::id == cl.astNodeId && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + nodes.insert(nodes.end(), result.begin(), result.end()); + } + + break; + } + + case INHERIT_BY: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + + for (const model::PythonInheritance& inh : + _db->query( + InhQuery::base == cl.id )) + { + model::PythonEntity cl = _db->query_value( + EntityQuery::id == inh.derived); + AstResult result = _db->query( + AstQuery::id == cl.astNodeId && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + nodes.insert(nodes.end(), result.begin(), result.end()); + } + + break; + } + + case DATA_MEMBER: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + + for (const model::PythonClassMember& mem : _db->query( + ClassMemQuery::classId == cl.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Attribute)) + { + for (const model::PythonVariable& var : _db->query( + VarQuery::id == mem.memberId)) + { + model::PythonAstNode astNode = queryPythonAstNode(std::to_string(var.astNodeId)); + if (astNode.location.range.end.line != model::Position::npos){ + nodes.push_back(astNode); + } + } + } + + break; + } + + case METHOD: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + + for (const model::PythonClassMember& mem : _db->query( + ClassMemQuery::classId == cl.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Method)) + { + for (const model::PythonFunction& func : _db->query( + FuncQuery::id == mem.memberId)) + { + nodes.push_back(queryPythonAstNode(std::to_string(func.astNodeId))); + } + } + + break; + } + + case NESTED_CLASS: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + + for (const model::PythonClassMember& mem : _db->query( + ClassMemQuery::classId == cl.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Class)) + { + for (const model::PythonClass& cl : _db->query( + ClassQuery::id == mem.memberId)) + { + nodes.push_back(queryPythonAstNode(std::to_string(cl.astNodeId))); + } + } + + break; + } + + case IMPORTED_SYMBOLS: + { + for (const model::PythonImport& imp : _db->query( + ModImpQuery::astNodeId == node.id && + ModImpQuery::importedSymbol != 0)) + { + EntityResult symb = _db->query( + EntityQuery::id == imp.importedSymbol); + + nodes.push_back(queryPythonAstNode(std::to_string(symb.begin()->astNodeId))); + } + + break; + } + } + + std::sort(nodes.begin(), nodes.end(), compareByValue); + + return_.reserve(nodes.size()); + _transaction([this, &return_, &nodes](){ + std::transform( + nodes.begin(), nodes.end(), + std::back_inserter(return_), + CreateAstNodeInfo(getVisibilities(nodes))); + }); + }); +} + +std::int32_t PythonServiceHandler::getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + return _transaction([&, this]() -> std::int32_t { + switch (referenceId_) + { + case DECLARATION: + return queryPythonAstNodeCount(astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + + case USAGE: + return queryPythonAstNodeCount(astNodeId_); + + case THIS_CALLS: + return queryCallsCount(astNodeId_); + + case CALLS_OF_THIS: + return queryPythonAstNodeCount(astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Usage); + + case CALLEE: + { + std::int32_t count = 0; + + std::set astNodeIds; + for (const model::PythonAstNode& call : queryCalls(astNodeId_)) + { + model::PythonAstNode node = queryPythonAstNode(std::to_string(call.id)); + astNodeIds.insert(node.id); + } + + if (!astNodeIds.empty()){ + count = _db->query_value( + AstQuery::id.in_range( + astNodeIds.begin(), astNodeIds.end()) && + AstQuery::astType == model::PythonAstNode::AstType::Declaration && + AstQuery::location.range.end.line != model::Position::npos).count; + } + + return count; + } + + case CALLER: + { + std::vector references; + getReferences(references, astNodeId_, CALLER, {}); + return references.size(); + } + + case PARAMETER: + return _db->query_value( + FuncQuery::qualifiedName == node.qualifiedName).count; + + case LOCAL_VAR: + return _db->query_value( + FuncQuery::qualifiedName == node.qualifiedName).count; + + case RETURN_TYPE: + { + FuncResult functions = _db->query( + FuncQuery::qualifiedName == node.qualifiedName); + + const model::PythonFunction& function = *functions.begin(); + + std::vector types = queryTypes(function); + + return types.size(); + } + + case TYPE: + { + VarResult varNodes = _db->query( + VarQuery::qualifiedName == node.qualifiedName); + + const model::PythonVariable& variable = *varNodes.begin(); + + std::vector types = queryTypes(variable); + + return types.size(); + } + + case INHERIT_FROM: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + return _db->query_value( + InhQuery::derived == cl.id).count; + } + + case INHERIT_BY: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + return _db->query_value( + InhQuery::base == cl.id).count; + } + + case DATA_MEMBER: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + return _db->query_value( + ClassMemQuery::classId == cl.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Attribute).count; + } + + case METHOD: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + return _db->query_value( + ClassMemQuery::classId == cl.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Method).count; + } + + case NESTED_CLASS: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + return _db->query_value( + ClassMemQuery::classId == cl.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Class).count; + } + + case IMPORTED_SYMBOLS: + return _db->query_value( + ModImpQuery::importer == node.location.file.object_id()).count; + + default: + return 0; + } + }); +} + +void PythonServiceHandler::getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) +{ + // TODO +} + +void PythonServiceHandler::getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) +{ + // TODO +} + +void PythonServiceHandler::getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) +{ + return_["Imports"] = IMPORTS; + return_["Classes"] = CLASSES; + return_["Functions"] = FUNCTIONS; + return_["Variables"] = VARIABLES; +} + +void PythonServiceHandler::getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) +{ + std::vector nodes; + + _transaction([&, this](){ + switch (referenceId_) + { + case CLASSES: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Class); + break; + + case VARIABLES: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Variable && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + break; + + case FUNCTIONS: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + (AstQuery::astType == model::PythonAstNode::AstType::Declaration)); + break; + + case IMPORTS: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Module); + break; + } + + std::sort(nodes.begin(), nodes.end(), compareByValue); + + return_.reserve(nodes.size()); + _transaction([this, &return_, &nodes](){ + std::transform( + nodes.begin(), nodes.end(), + std::back_inserter(return_), + CreateAstNodeInfo(getVisibilities(nodes))); + }); + }); +} + +std::int32_t PythonServiceHandler::getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) +{ + return _transaction([&, this]() -> std::int32_t { + switch (referenceId_) + { + case CLASSES: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Class); + break; + + case VARIABLES: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Variable && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + break; + + case FUNCTIONS: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + break; + + case IMPORTS: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::PythonAstNode::SymbolType::Module); + break; + + default: + return 0; + } + }); +} + +void PythonServiceHandler::getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) +{ + std::vector content; + + _transaction([&, this]() { + + //--- Load the file content and break it into lines ---// + + model::FilePtr file = _db->query_one( + FileQuery::id == std::stoull(range_.file)); + + if (!file->content.load()){ + return; + } + + std::istringstream s(file->content->content); + std::string line; + while (std::getline(s, line)){ + content.push_back(line); + } + + //--- Iterate over AST node elements ---// + + for (const model::PythonAstNode& node : _db->query( + AstQuery::location.file == std::stoull(range_.file) && + AstQuery::location.range.start.line >= range_.range.startpos.line && + AstQuery::location.range.end.line < range_.range.endpos.line && + AstQuery::location.range.end.line != model::Position::npos)) + { + if (node.astValue.empty()){ + continue; + } + + // Regular expression to find element position + std::string reg = "\\b" + node.astValue + "\\b"; + + for (std::size_t i = node.location.range.start.line - 1; + i < node.location.range.end.line && i < content.size(); + ++i) + { + std::regex words_regex(reg); + auto words_begin = std::sregex_iterator( + content[i].begin(), content[i].end(), + words_regex); + auto words_end = std::sregex_iterator(); + + for (std::sregex_iterator ri = words_begin; ri != words_end; ++ri) + { + SyntaxHighlight syntax; + syntax.range.startpos.line = i + 1; + syntax.range.startpos.column = ri->position() + 1; + syntax.range.endpos.line = i + 1; + syntax.range.endpos.column = + syntax.range.startpos.column + node.astValue.length(); + + std::string symbolClass = + "cm-" + model::symbolTypeToString(node.symbolType); + syntax.className = symbolClass + " " + + symbolClass + "-" + model::astTypeToString(node.astType); + + return_.push_back(std::move(syntax)); + } + } + } + }); +} + +bool PythonServiceHandler::compareByPosition( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs) +{ + return lhs.location.range.start < rhs.location.range.start; +} + +bool PythonServiceHandler::compareByValue( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs) +{ + return lhs.astValue < rhs.astValue; +} + +model::PythonAstNode PythonServiceHandler::queryPythonAstNode(const core::AstNodeId& astNodeId_) +{ + return _transaction([&, this](){ + model::PythonAstNode node; + + if (!_db->find(std::stoull(astNodeId_), node)) + { + core::InvalidId ex; + std::cout << "Invalid PythonAstNode ID: " << astNodeId_ << std::endl; + ex.__set_msg("Invalid PythonAstNode ID"); + ex.__set_nodeid(astNodeId_); + throw ex; + } + + return node; + }); +} + +std::vector PythonServiceHandler::queryPythonAstNodes( + const core::AstNodeId& astNodeId_, + const odb::query& query_) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + AstResult result = _db->query( + AstQuery::qualifiedName == node.qualifiedName && + AstQuery::location.range.end.line != model::Position::npos && + query_); + + return std::vector(result.begin(), result.end()); +} + +std::vector PythonServiceHandler::queryPythonAstNodesInFile( + const core::FileId& fileId_, + const odb::query& query_) +{ + AstResult result = _db->query( + AstQuery::location.file == std::stoull(fileId_) && query_); + + return std::vector(result.begin(), result.end()); +} + +std::uint32_t PythonServiceHandler::queryPythonAstNodeCountInFile( + const core::FileId& fileId_, + const odb::query& query_) +{ + return _db->query_value( + AstQuery::location.file == std::stoull(fileId_) && query_).count; +} + +std::vector PythonServiceHandler::queryDeclarations(const core::AstNodeId& astNodeId_) +{ + return queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Declaration); +} + +odb::query PythonServiceHandler::astCallsQuery(const model::PythonAstNode& astNode_) +{ + const model::Position& start = astNode_.location.range.start; + const model::Position& end = astNode_.location.range.end; + + return (AstQuery::location.file == astNode_.location.file.object_id() && + (AstQuery::astType == model::PythonAstNode::AstType::Usage || + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + // StartPos >= Pos + ((AstQuery::location.range.start.line == start.line && + AstQuery::location.range.start.column >= start.column) || + AstQuery::location.range.start.line > start.line) && + // Pos > EndPos + ((AstQuery::location.range.end.line == end.line && + AstQuery::location.range.end.column < end.column) || + AstQuery::location.range.end.line < end.line))); +} + +std::vector PythonServiceHandler::queryCalls(const core::AstNodeId& astNodeId_) +{ + std::vector nodes = queryDeclarations(astNodeId_); + + if (nodes.empty()){ + return nodes; + } + + model::PythonAstNode node = nodes.front(); + AstResult result = _db->query(astCallsQuery(node)); + + return std::vector(result.begin(), result.end()); +} + +std::size_t PythonServiceHandler::queryPythonAstNodeCount( + const core::AstNodeId& astNodeId_, + const odb::query& query_) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + model::PythonAstCount q = _db->query_value( + AstQuery::qualifiedName == node.qualifiedName && + AstQuery::location.range.end.line != model::Position::npos && + query_); + + return q.count; +} + +std::size_t PythonServiceHandler::queryCallsCount(const core::AstNodeId& astNodeId_) +{ + std::vector nodes = queryDeclarations(astNodeId_); + + if (nodes.empty()){ + return std::size_t(0); + } + + model::PythonAstNode node = nodes.front(); + + return _db->query_value(astCallsQuery(node)).count; +} + +std::vector PythonServiceHandler::queryTypes(const model::PythonEntity& entity) +{ + std::vector result; + + for(const model::PythonType& type : _db->query(TypeQuery::symbol == entity.id)){ + ClassResult cl = _db->query(ClassQuery::id == type.type); + result.push_back(*cl.begin()); + } + + return result; +} + +model::PythonEntity PythonServiceHandler::queryPythonEntity(const model::PythonEntityId& id) +{ + EntityResult entities = _db->query(EntityQuery::id == id); + return *entities.begin(); +} + +model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const std::string& qualifiedName) +{ + EntityResult entities = _db->query(EntityQuery::qualifiedName == qualifiedName); + return *entities.begin(); +} + +std::map PythonServiceHandler::getVisibilities( + const std::vector& nodes_) +{ + std::map visibilities; + + for (const model::PythonAstNode& node : nodes_) + { + switch(node.symbolType){ + case model::PythonAstNode::SymbolType::Variable: + case model::PythonAstNode::SymbolType::Function: + case model::PythonAstNode::SymbolType::Class: + model::PythonEntity entity = queryPythonEntityByAstNode(node.qualifiedName); + visibilities[node.id] = entity.visibility; + break; + } + } + + return visibilities; +} + +bool PythonServiceHandler::isGeneratedVariable(const model::PythonAstNode& node_) const +{ + const std::vector generatedVariables {"__dict__", "__doc__", "__module__"}; + return std::any_of(generatedVariables.begin(), generatedVariables.end(), + [&](const std::string& name){ return node_.qualifiedName.find(name) != std::string::npos; }) && + node_.astType == model::PythonAstNode::AstType::Declaration; +} + +} +} +} diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js new file mode 100644 index 000000000..344c17f9e --- /dev/null +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -0,0 +1,261 @@ +require([ + 'codecompass/model', + 'codecompass/viewHandler', + 'codecompass/util'], +function (model, viewHandler, util){ + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + function createLabel(astNodeInfo) { + return '' + + astNodeInfo.range.range.startpos.line + ':' + + astNodeInfo.range.range.startpos.column + ': ' + + astNodeInfo.astNodeValue + + ''; + } + + function getCssClass(astNodeInfo) { + var tags = astNodeInfo.tags; + + return tags.indexOf('public') > -1 ? 'icon-visibility icon-public' : + tags.indexOf('private') > -1 ? 'icon-visibility icon-private' : + tags.indexOf('semiprivate') > -1 ? 'icon-visibility icon-protected' : + null; + } + + function groupReferencesByVisibilities(references, parentNode, nodeInfo){ + var res = []; + var visibilities = ['public', 'semiprivate', 'private'] + + visibilities.forEach(function (visibility){ + var nodes = references.filter(function (reference){ + return reference.tags.indexOf(visibility) > -1; + }); + + if (nodes.length === 0){ + return; + } + + var visibility_icon_name = visibility === 'semiprivate' ? 'protected' : visibility; + + res.push({ + id: nodeInfo.id + visibility + parentNode.refType, + name: createReferenceCountLabel(visibility, nodes.length), + refType: parentNode.refType, + hasChildren: true, + cssClass: 'icon-visibility icon-' + visibility_icon_name, + getChildren: function (){ + var res = []; + nodes.forEach(function (reference){ + res.push({ + id: visibility + reference.id, + name: createLabel(reference), + refType: parentNode.refType, + nodeInfo: reference, + hasChildren: false, + cssClass: getCssClass(reference) + }); + }); + return res; + } + }); + }); + + return res; + } + + function createRootNode(elementInfo) { + var rootLabel + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.symbolType + : 'File') + + ''; + + var rootValue + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.astNodeValue + : elementInfo.name) + + ''; + + var label = '' + + rootLabel + ': ' + rootValue + + ''; + + return { + id : 'root', + name : label, + cssClass : 'icon-info', + hasChildren : true, + getChildren : function () { + return this._store.query({ parent : 'root' }); + } + }; + } + + function createReferenceCountLabel(label, count) { + return label + '(' + count + ')'; + } + + function loadReferenceNodes(parentNode, nodeInfo, refTypes) { + var res = []; + var fileGroupsId = []; + + var references = model.pythonservice.getReferences(nodeInfo.id, parentNode.refType); + + if (parentNode.refType === refTypes['Method'] || + parentNode.refType === refTypes['Data member']){ + return groupReferencesByVisibilities(references, parentNode, nodeInfo); + } + + references.forEach(function (reference) { + if (parentNode.refType === refTypes['Caller'] || + parentNode.refType === refTypes['Usage']) { + var fileId = reference.range.file; + if (fileGroupsId[fileId]){ + return; + } + + fileGroupsId[fileId] = parentNode.refType + fileId + reference.id; + + var referenceInFile = references.filter(function (ref) { + return reference.range.file === fileId; + }); + + var fileInfo = model.project.getFileInfo(fileId); + res.push({ + id: fileGroupsId[fileId], + name: createReferenceCountLabel(fileInfo.name, referenceInFile.length), + refType: parentNode.refType, + hasChildren: true, + cssClass: util.getIconClass(fileInfo.path), + getChildren: function () { + var that = this; + var res = []; + + referenceInFile.forEach(function (ref) { + if (parentNode.refType === refTypes['Caller']) { + ref.push({ + id: ref.id, + name: createLabel(ref), + nodeInfo: ref, + refType: parentNode.refType, + cssClass: 'icon icon-Method', + hasChildren: true, + getChildren: function () { + var res = []; + + var refCount = model.pythonservice.getReferenceCount(ref.id, parentNode.refType); + + if (refCount > 0) { + res.push({ + id: 'Caller-' + ref.id, + name: createReferenceCountLabel(parentNode.name, refCount), + nodeInfo: ref, + refType: parentNode.refType, + cssClass: parentNode.cssClass, + hasChildren: true, + getChildren: parentNode.getChildren + }); + } + + var calls = model.pythonservice.getReferences(this.nodeInfo.id, refTypes['This calls']); + + calls.forEach(function (call) { + if (call.entityHash === nodeInfo.entityHash) + res.push({ + name: createLabel(call), + refType: parentNode.refType, + nodeInfo: call, + hasChildren: false, + cssClass: getCssClass(call) + }); + }); + + return res; + } + }); + } else if (parentNode.refType === refTypes['Usage']) { + res.push({ + id: fileGroupsId[fileId] + ref.id, + name: createLabel(ref), + refType: parentNode.refType, + nodeInfo: ref, + hasChildren: false, + cssClass: getCssClass(ref) + }); + } + }); + return res; + } + }); + } else { + res.push({ + name: createLabel(reference), + refType: parentNode.refType, + nodeInfo: reference, + hasChildren: false, + cssClass: getCssClass(reference) + }); + } + }); + + return res; + } + + var pythonInfoTree = { + id: 'python-info-tree', + render : function (elementInfo){ + var ret = []; + + ret.push(createRootNode(elementInfo)); + + if (elementInfo instanceof AstNodeInfo){ + var properties = model.pythonservice.getProperties(elementInfo.id); + + for (var propertyName in properties) { + var propertyId = propertyName.replace(/ /g, '-'); + var label = + '' + propertyName + ': ' + + '' + properties[propertyName] +''; + + ret.push({ + name: label, + parent: 'root', + nodeInfo: elementInfo, + cssClass: 'icon-' + propertyId, + hasChildren: false + }); + } + + var refTypes = model.pythonservice.getReferenceTypes(elementInfo.id); + + for (var refType in refTypes) { + var refCount = model.pythonservice.getReferenceCount(elementInfo.id, refTypes[refType]); + + if (refCount > 0){ + ret.push({ + name: createReferenceCountLabel(refType, refCount), + parent: 'root', + refType: refTypes[refType], + cssClass: 'icon-' + refType.replace(/ /g, '-'), + hasChildren: true, + getChildren: function () { + return loadReferenceNodes(this, elementInfo, refTypes); + } + }); + } + } + } else if (elementInfo instanceof FileInfo) { + + } + + return ret; + } + }; + + viewHandler.registerModule(pythonInfoTree, { + type : viewHandler.moduleType.InfoTree, + service : model.pythonservice + }); +}); diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js new file mode 100644 index 000000000..a69a0248e --- /dev/null +++ b/plugins/python/webgui/js/pythonMenu.js @@ -0,0 +1,124 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/astHelper', + 'codecompass/model', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, viewHandler){ + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + var getdefinition = { + id : 'python-text-getdefinition', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Jump to definition', + accelKey : 'ctrl - click', + onClick : function () { + if (!nodeInfo || !fileInfo) { + return; + } + + // var languageService = model.getLanguageService(fileInfo.type); + astHelper.jumpToDef(nodeInfo.id, model.pythonservice); + } + }); + } + }; + + viewHandler.registerModule(getdefinition, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infoTree = { + id : 'python-text-infotree', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Info Tree', + onClick : function () { + if (!nodeInfo || !fileInfo) { + return; + } + + topic.publish('codecompass/infotree', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + } + }); + } + }; + + viewHandler.registerModule(infoTree, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infobox = { + id : 'python-text-infobox', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Documentation', + onClick : function () { + topic.publish('codecompass/documentation', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + } + }); + } + }; + + viewHandler.registerModule(infobox, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var diagrams = { + id : 'python-text-diagrams', + render : function (nodeInfo, fileInfo) { + if (!nodeInfo || !fileInfo) + return; + + var submenu = new Menu(); + + // var diagramTypes = model.pythonservice.getDiagramTypes(nodeInfo.id); + // for (diagramType in diagramTypes) + // submenu.addChild(new MenuItem({ + // label : diagramType, + // type : diagramType, + // onClick : function () { + // var that = this; + + // topic.publish('codecompass/openDiagram', { + // handler : 'python-ast-diagram', + // diagramType : diagramTypes[that.type], + // node : nodeInfo.id + // }); + // } + // })); + + submenu.addChild(new MenuItem({ + label : "CodeBites", + onClick : function () { + topic.publish('codecompass/codebites', { + node : nodeInfo + }); + } + })); + + //if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(diagrams, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); +}); \ No newline at end of file diff --git a/util/include/util/odbtransaction.h b/util/include/util/odbtransaction.h index 7640c3d14..c28a78c52 100644 --- a/util/include/util/odbtransaction.h +++ b/util/include/util/odbtransaction.h @@ -176,7 +176,7 @@ void persistAll(Cont& cont_, std::shared_ptr db_) << item->toString(); LOG(warning) << ex.what() << std::endl - << "Further changes in this transaction will be ignored!"; + << "Further changes in this transaction will be ignored! -> " << item->toString(); } catch (const odb::database_exception& ex) { @@ -190,7 +190,7 @@ void persistAll(Cont& cont_, std::shared_ptr db_) } #endif - LOG(error) << ex.what() << std::endl; + LOG(error) << ex.what() << " -> " << item->toString() << std::endl; throw; } } diff --git a/util/src/dynamiclibrary.cpp b/util/src/dynamiclibrary.cpp index 251af2268..4d42c64ef 100644 --- a/util/src/dynamiclibrary.cpp +++ b/util/src/dynamiclibrary.cpp @@ -37,7 +37,7 @@ DynamicLibrary::DynamicLibrary(const std::string& path_) throw std::runtime_error(ss.str()); } #else - _handle = ::dlopen(path_.c_str(), RTLD_NOW); + _handle = ::dlopen(path_.c_str(), RTLD_NOW | RTLD_GLOBAL); if (!_handle) { const char *dlError = ::dlerror(); diff --git a/webgui/scripts/codecompass/view/codeBites.js b/webgui/scripts/codecompass/view/codeBites.js index 8d1be8f23..1b0c6402f 100644 --- a/webgui/scripts/codecompass/view/codeBites.js +++ b/webgui/scripts/codecompass/view/codeBites.js @@ -348,7 +348,12 @@ function (declare, array, dom, style, topic, on, ContentPane, ResizeHandle, fPos.file = this.astNodeInfo.range.file; var astNodeInfo = languageService.getAstNodeInfoByPosition(fPos); - var refTypes = languageService.getReferenceTypes(astNodeInfo.id); + var service = model.getLanguageService(fileInfo.type); + if(service){ + var refTypes = service.getReferenceTypes(astNodeInfo.id); + } else { + return; + } var defAstNodeInfo = languageService.getReferences( astNodeInfo.id, refTypes["Definition"])[0]; diff --git a/webgui/scripts/codecompass/view/component/Text.js b/webgui/scripts/codecompass/view/component/Text.js index fcc139f95..1a99b4174 100644 --- a/webgui/scripts/codecompass/view/component/Text.js +++ b/webgui/scripts/codecompass/view/component/Text.js @@ -507,10 +507,14 @@ function (declare, domClass, dom, style, query, topic, ContentPane, Dialog, return; var service = model.getLanguageService(fileInfo.type); - var refTypes = service.getReferenceTypes(astNodeInfo.id); - var usages = service.getReferences( - astNodeInfo.id, - refTypes['Usage']); + if(service){ + var refTypes = service.getReferenceTypes(astNodeInfo.id); + var usages = service.getReferences( + astNodeInfo.id, + refTypes['Usage']); + } else { + var usages = []; + } this.clearSelection();