From df1f17dd1d68af031be3f92b346a1d4404b4f2a6 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Fri, 30 Oct 2020 14:23:59 +0100 Subject: [PATCH 01/39] Python plugin introduced --- CMakeLists.txt | 2 +- plugins/python/CMakeLists.txt | 5 + plugins/python/model/CMakeLists.txt | 17 + .../model/include/model/pythonastnode.h | 128 +++ .../python/model/include/model/pythonclass.h | 77 ++ .../model/include/model/pythondocumentation.h | 47 + .../python/model/include/model/pythonentity.h | 35 + .../model/include/model/pythonfunction.h | 51 + .../python/model/include/model/pythonimport.h | 49 + .../model/include/model/pythoninheritance.h | 44 + .../python/model/include/model/pythontype.h | 26 + .../model/include/model/pythonvariable.h | 27 + plugins/python/parser/CMakeLists.txt | 23 + .../include/pythonparser/pythonparser.h | 28 + plugins/python/parser/src/my_ast/__init__.py | 7 + plugins/python/parser/src/my_ast/base_data.py | 66 ++ .../parser/src/my_ast/built_in_functions.py | 612 +++++++++++ .../parser/src/my_ast/built_in_operators.py | 324 ++++++ .../parser/src/my_ast/built_in_types.py | 174 ++++ .../python/parser/src/my_ast/class_data.py | 55 + .../src/my_ast/class_init_declaration.py | 16 + .../parser/src/my_ast/class_preprocessor.py | 84 ++ .../parser/src/my_ast/common/__init__.py | 1 + .../parser/src/my_ast/common/file_position.py | 20 + .../parser/src/my_ast/common/hashable_list.py | 31 + .../parser/src/my_ast/common/location.py | 8 + .../parser/src/my_ast/common/parser_tree.py | 30 + .../parser/src/my_ast/common/position.py | 36 + .../parser/src/my_ast/common/unique_list.py | 23 + .../python/parser/src/my_ast/common/utils.py | 18 + plugins/python/parser/src/my_ast/db_test.log | 27 + plugins/python/parser/src/my_ast/file_info.py | 59 ++ .../python/parser/src/my_ast/function_data.py | 41 + .../src/my_ast/function_symbol_collector.py | 64 ++ .../function_symbol_collector_factory.py | 14 + .../python/parser/src/my_ast/import_finder.py | 27 + .../parser/src/my_ast/import_preprocessor.py | 129 +++ plugins/python/parser/src/my_ast/logger.py | 16 + plugins/python/parser/src/my_ast/main.py | 43 + .../src/my_ast/member_access_collector.py | 210 ++++ .../parser/src/my_ast/parse_exception.py | 20 + plugins/python/parser/src/my_ast/parser.py | 127 +++ .../parser/src/my_ast/persistence/__init__.py | 2 + .../parser/src/my_ast/persistence/base_dto.py | 18 + .../src/my_ast/persistence/build_action.py | 11 + .../my_ast/persistence/build_source_target.py | 14 + .../src/my_ast/persistence/class_dto.py | 23 + .../src/my_ast/persistence/documented_dto.py | 3 + .../parser/src/my_ast/persistence/file.py | 38 + .../src/my_ast/persistence/file_content.py | 3 + .../my_ast/persistence/file_content_dto.py | 3 + .../parser/src/my_ast/persistence/file_dto.py | 23 + .../src/my_ast/persistence/function_dto.py | 12 + .../src/my_ast/persistence/import_dto.py | 17 + .../src/my_ast/persistence/persistence.py | 48 + .../src/my_ast/persistence/variable_dto.py | 5 + .../placeholder_function_declaration_cache.py | 49 + .../parser/src/my_ast/preprocessed_data.py | 8 + .../parser/src/my_ast/preprocessed_file.py | 59 ++ .../src/my_ast/preprocessed_function.py | 19 + .../src/my_ast/preprocessed_variable.py | 19 + .../python/parser/src/my_ast/python_parser.py | 30 + plugins/python/parser/src/my_ast/scope.py | 177 ++++ .../python/parser/src/my_ast/scope_manager.py | 383 +++++++ .../parser/src/my_ast/symbol_collector.py | 881 ++++++++++++++++ .../src/my_ast/symbol_collector_interface.py | 27 + .../python/parser/src/my_ast/symbol_finder.py | 10 + plugins/python/parser/src/my_ast/test.log | 27 + plugins/python/parser/src/my_ast/type_data.py | 52 + .../parser/src/my_ast/type_deduction.py | 507 +++++++++ .../python/parser/src/my_ast/variable_data.py | 77 ++ plugins/python/parser/src/pythonparser.cpp | 678 ++++++++++++ plugins/python/service/CMakeLists.txt | 28 + .../service/include/service/pythonservice.h | 239 +++++ plugins/python/service/src/pythonservice.cpp | 978 ++++++++++++++++++ plugins/python/webgui/js/pythonInfoTree.js | 14 + plugins/python/webgui/js/pythonMenu.js | 11 + 77 files changed, 7333 insertions(+), 1 deletion(-) create mode 100644 plugins/python/CMakeLists.txt create mode 100644 plugins/python/model/CMakeLists.txt create mode 100644 plugins/python/model/include/model/pythonastnode.h create mode 100644 plugins/python/model/include/model/pythonclass.h create mode 100644 plugins/python/model/include/model/pythondocumentation.h create mode 100644 plugins/python/model/include/model/pythonentity.h create mode 100644 plugins/python/model/include/model/pythonfunction.h create mode 100644 plugins/python/model/include/model/pythonimport.h create mode 100644 plugins/python/model/include/model/pythoninheritance.h create mode 100644 plugins/python/model/include/model/pythontype.h create mode 100644 plugins/python/model/include/model/pythonvariable.h create mode 100644 plugins/python/parser/CMakeLists.txt create mode 100644 plugins/python/parser/include/pythonparser/pythonparser.h create mode 100644 plugins/python/parser/src/my_ast/__init__.py create mode 100644 plugins/python/parser/src/my_ast/base_data.py create mode 100644 plugins/python/parser/src/my_ast/built_in_functions.py create mode 100644 plugins/python/parser/src/my_ast/built_in_operators.py create mode 100644 plugins/python/parser/src/my_ast/built_in_types.py create mode 100644 plugins/python/parser/src/my_ast/class_data.py create mode 100644 plugins/python/parser/src/my_ast/class_init_declaration.py create mode 100644 plugins/python/parser/src/my_ast/class_preprocessor.py create mode 100644 plugins/python/parser/src/my_ast/common/__init__.py create mode 100644 plugins/python/parser/src/my_ast/common/file_position.py create mode 100644 plugins/python/parser/src/my_ast/common/hashable_list.py create mode 100644 plugins/python/parser/src/my_ast/common/location.py create mode 100644 plugins/python/parser/src/my_ast/common/parser_tree.py create mode 100644 plugins/python/parser/src/my_ast/common/position.py create mode 100644 plugins/python/parser/src/my_ast/common/unique_list.py create mode 100644 plugins/python/parser/src/my_ast/common/utils.py create mode 100644 plugins/python/parser/src/my_ast/db_test.log create mode 100644 plugins/python/parser/src/my_ast/file_info.py create mode 100644 plugins/python/parser/src/my_ast/function_data.py create mode 100644 plugins/python/parser/src/my_ast/function_symbol_collector.py create mode 100644 plugins/python/parser/src/my_ast/function_symbol_collector_factory.py create mode 100644 plugins/python/parser/src/my_ast/import_finder.py create mode 100644 plugins/python/parser/src/my_ast/import_preprocessor.py create mode 100644 plugins/python/parser/src/my_ast/logger.py create mode 100644 plugins/python/parser/src/my_ast/main.py create mode 100644 plugins/python/parser/src/my_ast/member_access_collector.py create mode 100644 plugins/python/parser/src/my_ast/parse_exception.py create mode 100644 plugins/python/parser/src/my_ast/parser.py create mode 100644 plugins/python/parser/src/my_ast/persistence/__init__.py create mode 100644 plugins/python/parser/src/my_ast/persistence/base_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/build_action.py create mode 100644 plugins/python/parser/src/my_ast/persistence/build_source_target.py create mode 100644 plugins/python/parser/src/my_ast/persistence/class_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/documented_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/file.py create mode 100644 plugins/python/parser/src/my_ast/persistence/file_content.py create mode 100644 plugins/python/parser/src/my_ast/persistence/file_content_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/file_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/function_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/import_dto.py create mode 100644 plugins/python/parser/src/my_ast/persistence/persistence.py create mode 100644 plugins/python/parser/src/my_ast/persistence/variable_dto.py create mode 100644 plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py create mode 100644 plugins/python/parser/src/my_ast/preprocessed_data.py create mode 100644 plugins/python/parser/src/my_ast/preprocessed_file.py create mode 100644 plugins/python/parser/src/my_ast/preprocessed_function.py create mode 100644 plugins/python/parser/src/my_ast/preprocessed_variable.py create mode 100644 plugins/python/parser/src/my_ast/python_parser.py create mode 100644 plugins/python/parser/src/my_ast/scope.py create mode 100644 plugins/python/parser/src/my_ast/scope_manager.py create mode 100644 plugins/python/parser/src/my_ast/symbol_collector.py create mode 100644 plugins/python/parser/src/my_ast/symbol_collector_interface.py create mode 100644 plugins/python/parser/src/my_ast/symbol_finder.py create mode 100644 plugins/python/parser/src/my_ast/test.log create mode 100644 plugins/python/parser/src/my_ast/type_data.py create mode 100644 plugins/python/parser/src/my_ast/type_deduction.py create mode 100644 plugins/python/parser/src/my_ast/variable_data.py create mode 100644 plugins/python/parser/src/pythonparser.cpp create mode 100644 plugins/python/service/CMakeLists.txt create mode 100644 plugins/python/service/include/service/pythonservice.h create mode 100644 plugins/python/service/src/pythonservice.cpp create mode 100644 plugins/python/webgui/js/pythonInfoTree.js create mode 100644 plugins/python/webgui/js/pythonMenu.js diff --git a/CMakeLists.txt b/CMakeLists.txt index d4e81c7d0..7e71ab356 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ include(Functions.cmake) # Do some sanity check on the testing setup and enable testing if applicable. include(Testing.cmake) -find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread) +find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread python) find_package(Java REQUIRED) find_package(Odb REQUIRED) find_package(Threads REQUIRED) diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt new file mode 100644 index 000000000..166d56255 --- /dev/null +++ b/plugins/python/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(parser) +add_subdirectory(model) +add_subdirectory(service) + +install_webplugin(webgui) \ No newline at end of file diff --git a/plugins/python/model/CMakeLists.txt b/plugins/python/model/CMakeLists.txt new file mode 100644 index 000000000..bb4200716 --- /dev/null +++ b/plugins/python/model/CMakeLists.txt @@ -0,0 +1,17 @@ +set(ODB_SOURCES + include/model/pythonastnode.h + include/model/pythonclass.h + include/model/pythondocumentation.h + include/model/pythonentity.h + include/model/pythonfunction.h + include/model/pythonimport.h + include/model/pythoninheritance.h + include/model/pythontype.h + include/model/pythonvariable.h) + +generate_odb_files("${ODB_SOURCES}") + +add_odb_library(pythonmodel ${ODB_CXX_SOURCES}) +target_link_libraries(pythonmodel model) + +install_sql() diff --git a/plugins/python/model/include/model/pythonastnode.h b/plugins/python/model/include/model/pythonastnode.h new file mode 100644 index 000000000..d7e9fa1d6 --- /dev/null +++ b/plugins/python/model/include/model/pythonastnode.h @@ -0,0 +1,128 @@ +#ifndef CC_MODEL_PYTHONASTNODE_H +#define CC_MODEL_PYTHONASTNODE_H + +#include + +#include +#include +#include + +#include + +namespace cc +{ +namespace model +{ + +typedef std::uint64_t PythonAstNodeId; + +#pragma db object +struct PythonAstNode +{ + enum class SymbolType + { + Variable, + Function, + Class, + Module, + Other + }; + + enum class AstType + { + Declaration, + Usage, + Other + }; + + virtual ~PythonAstNode() {} + + #pragma db id + PythonAstNodeId id = 0; + + std::string astValue; + + std::string qualifiedName; + + #pragma db null + FileLoc location; + + SymbolType symbolType = SymbolType::Other; + + AstType astType = AstType::Other; + + std::string toString() const; + + bool operator< (const PythonAstNode& other) const { return id < other.id; } + bool operator==(const PythonAstNode& other) const { return id == other.id; } +}; + +typedef std::shared_ptr PythonAstNodePtr; + +inline std::string symbolTypeToString(PythonAstNode::SymbolType type_) +{ + switch (type_) + { + case PythonAstNode::SymbolType::Variable: return "Variable"; + case PythonAstNode::SymbolType::Function: return "Function"; + case PythonAstNode::SymbolType::Class: return "Class"; + case PythonAstNode::SymbolType::Module: return "Module"; + case PythonAstNode::SymbolType::Other: return "Other"; + } + + return std::string(); +} + +inline std::string astTypeToString(PythonAstNode::AstType type) +{ + switch (type) { + case PythonAstNode::AstType::Usage: return "Usage"; + case PythonAstNode::AstType::Declaration: return "Declaration"; + case PythonAstNode::AstType::Other: return "Other"; + } + + return std::string(); +} + +inline std::string PythonAstNode::toString() const +{ + return std::string("PythonAstNode") + .append("\nid = ").append(std::to_string(id)) + .append("\nastValue = ").append(astValue) + .append("\nqualifiedName = ").append(qualifiedName) + .append("\nlocation = ").append(location.file->path).append(" (") + .append(std::to_string( + static_cast(location.range.start.line))).append(":") + .append(std::to_string( + static_cast(location.range.start.column))).append(" - ") + .append(std::to_string( + static_cast(location.range.end.line))).append(":") + .append(std::to_string( + static_cast(location.range.end.column))).append(")") + .append("\nsymbolType = ").append(symbolTypeToString(symbolType)) + .append("\nastType = ").append(astTypeToString(astType)); +} + +#pragma db view \ + object(PythonAstNode) object(File = LocFile : PythonAstNode::location.file) \ + query ((?) + "GROUP BY" + LocFile::id + "ORDER BY" + LocFile::id) +struct PythonAstCountGroupByFiles +{ + #pragma db column(LocFile::id) + FileId file; + + #pragma db column("count(" + PythonAstNode::id + ")") + std::size_t count; +}; + +#pragma db view object(PythonAstNode) +struct PythonAstCount +{ + #pragma db column("count(" + PythonAstNode::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonclass.h b/plugins/python/model/include/model/pythonclass.h new file mode 100644 index 000000000..3e5795446 --- /dev/null +++ b/plugins/python/model/include/model/pythonclass.h @@ -0,0 +1,77 @@ +#ifndef CC_MODEL_PYTHONCLASS_H +#define CC_MODEL_PYTHONCLASS_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonClass : PythonEntity +{ + std::string toString() const + { + return std::string("PythonClass") + .append("\nid = ").append(std::to_string(id)) + .append("\nqualifiedName = ").append(qualifiedName); + } +}; + +typedef std::shared_ptr PythonClassPtr; + +#pragma db object +struct PythonClassMember +{ + enum Kind + { + Attribute, + Method, + Class + }; + + #pragma db id auto + int id; + + #pragma db unique + PythonAstNodeId astNodeId; + + PythonEntityId memberId; + PythonEntityId classId; + + Kind kind; + bool staticMember = false; + + std::string toString() const + { + return std::string("PythonClassMember") + .append("\nid = ").append(std::to_string(id)) + .append("\nmemberId = ").append(std::to_string(memberId)) + .append("\nclassId = ").append(std::to_string(classId)) + .append("\nstaticMember = ").append(std::to_string(staticMember)) + .append("\nkind = ").append(kind == Attribute ? "Attribute" : + kind == Method ? "Method" : "Class"); + } +}; + +typedef std::shared_ptr PythonClassMemberPtr; + +#pragma db view object(PythonClass) +struct PythonClassCount +{ + #pragma db column("count(" + PythonClass::id + ")") + std::size_t count; +}; + +#pragma db view object(PythonClassMember) +struct PythonClassMemberCount +{ + #pragma db column("count(" + PythonClassMember::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythondocumentation.h b/plugins/python/model/include/model/pythondocumentation.h new file mode 100644 index 000000000..a2900aee7 --- /dev/null +++ b/plugins/python/model/include/model/pythondocumentation.h @@ -0,0 +1,47 @@ +#ifndef CC_MODULE_PYTHONDOCUMENTATION_H +#define CC_MODULE_PYTHONDOCUMENTATION_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonDocumentation +{ + enum Kind + { + Function, + Class, + Module + }; + + #pragma db id auto + int id; + + std::string documentation; + + PythonEntityId documented; + + Kind documentationKind; + + std::string toString() const + { + return std::string("PythonPythonDocumentation") + .append("\nid = ").append(std::to_string(id)) + .append("\ndocumentation = ").append(documentation) + .append("\ndocumented = ").append(std::to_string(documented)) + .append("\ndocumentationKind = ").append( + documentationKind == Function ? "Function" : + documentationKind == Class ? "Class" : "Module"); + } +}; + +typedef std::shared_ptr PythonDocumentationPtr; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonentity.h b/plugins/python/model/include/model/pythonentity.h new file mode 100644 index 000000000..cbc8245c8 --- /dev/null +++ b/plugins/python/model/include/model/pythonentity.h @@ -0,0 +1,35 @@ +#ifndef CC_MODEL_PYTHONENTITY_H +#define CC_MODEL_PYTHONENTITY_H + +#include + +#include "pythonastnode.h" + +namespace cc +{ +namespace model +{ + +typedef std::uint64_t PythonEntityId; + +#pragma db object polymorphic +struct PythonEntity +{ + virtual ~PythonEntity() {} + + #pragma db id auto + PythonEntityId id; + + #pragma db unique + PythonAstNodeId astNodeId; + + std::string name; + std::string qualifiedName; +}; + +typedef std::shared_ptr PythonEntityPtr; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonfunction.h b/plugins/python/model/include/model/pythonfunction.h new file mode 100644 index 000000000..0095d3a6e --- /dev/null +++ b/plugins/python/model/include/model/pythonfunction.h @@ -0,0 +1,51 @@ +#ifndef CC_MODEL_PYTHONFUNCTION_H +#define CC_MODEL_PYTHONFUNCTION_H + +#include + +#include "pythonvariable.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonFunction : PythonEntity +{ + #pragma db on_delete(cascade) + std::vector> parameters; + + #pragma db on_delete(cascade) + std::vector> locals; + + std::string toString() const + { + return std::string("PythonFunction") + .append("\nid = ").append(std::to_string(id)) + .append("\nqualifiedName = ").append(qualifiedName); + } +}; + +typedef std::shared_ptr PythonFunctionPtr; + +#pragma db view \ + object(PythonFunction) object(PythonVariable = Parameters : PythonFunction::parameters) +struct PythonFunctionParamCount +{ + #pragma db column("count(" + Parameters::id + ")") + std::size_t count; +}; + +#pragma db view \ + object(PythonFunction) object(PythonVariable = Locals : PythonFunction::locals) +struct PythonFunctionLocalCount +{ + #pragma db column("count(" + Locals::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonimport.h b/plugins/python/model/include/model/pythonimport.h new file mode 100644 index 000000000..7bf792b7a --- /dev/null +++ b/plugins/python/model/include/model/pythonimport.h @@ -0,0 +1,49 @@ +#ifndef CC_MODEL_PYTHONIMPORT_H +#define CC_MODEL_PYTHONIMPORT_H + +#include + +#include + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonImport +{ + #pragma db id auto + int id; + + #pragma db not_null + #pragma db on_delete(cascade) + odb::lazy_shared_ptr importer; + + #pragma db not_null + #pragma db on_delete(cascade) + odb::lazy_shared_ptr imported; + + PythonEntityId importedSymbol; + + std::string toString() const + { + return std::string("PythonModuleImport") + .append("\nid = ").append(std::to_string(id)) + .append("\nimporter = ").append(std::to_string(importer->id)) + .append("\nimported = ").append(std::to_string(imported->id)) + .append("\nimported symbols =").append(std::to_string(importedSymbol)); + } + +#pragma db index member(importer) +#pragma db index member(imported) +}; + +typedef std::shared_ptr PythonImportPtr; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythoninheritance.h b/plugins/python/model/include/model/pythoninheritance.h new file mode 100644 index 000000000..c4713bbfb --- /dev/null +++ b/plugins/python/model/include/model/pythoninheritance.h @@ -0,0 +1,44 @@ +#ifndef CC_MODEL_PYTHONINHERITANCE_H +#define CC_MODEL_PYTHONINHERITANCE_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonInheritance +{ + #pragma db id auto + int id; + + PythonEntityId derived; + PythonEntityId base; + + std::string toString() const + { + return std::string("PythonInheritance") + .append("\nid = ").append(std::to_string(id)) + .append("\nderived = ").append(std::to_string(derived)) + .append("\nbase = ").append(std::to_string(base)); + } + +#pragma db index member(derived) +#pragma db index member(base) +}; + +typedef std::shared_ptr PythonInheritancePtr; + +#pragma db view object(PythonInheritance) +struct PythonInheritanceCount +{ + #pragma db column("count(" + PythonInheritance::id + ")") + std::size_t count; +}; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythontype.h b/plugins/python/model/include/model/pythontype.h new file mode 100644 index 000000000..f8dcfeda0 --- /dev/null +++ b/plugins/python/model/include/model/pythontype.h @@ -0,0 +1,26 @@ +#ifndef CC_MODEL_PYTHONTYPE_H +#define CC_MODEL_PYTHONTYPE_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonType +{ + #pragma db id auto + std::uint64_t id; + + PythonEntityId type; + PythonEntityId symbol; +}; + +typedef std::shared_ptr PythonTypePtr; + +} +} + +#endif diff --git a/plugins/python/model/include/model/pythonvariable.h b/plugins/python/model/include/model/pythonvariable.h new file mode 100644 index 000000000..0a745d395 --- /dev/null +++ b/plugins/python/model/include/model/pythonvariable.h @@ -0,0 +1,27 @@ +#ifndef CC_MODEL_PYTHONVARIABLE_H +#define CC_MODEL_PYTHONVARIABLE_H + +#include "pythonentity.h" + +namespace cc +{ +namespace model +{ + +#pragma db object +struct PythonVariable : PythonEntity +{ + std::string toString() const + { + return std::string("PythonVariable") + .append("\nid = ").append(std::to_string(id)) + .append("\nqualifiedName = ").append(qualifiedName); + } +}; + +typedef std::shared_ptr PythonVariablePtr; + +} +} + +#endif diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt new file mode 100644 index 000000000..e3709df09 --- /dev/null +++ b/plugins/python/parser/CMakeLists.txt @@ -0,0 +1,23 @@ +find_package(PythonLibs REQUIRED) + +include_directories(${PYTHON_INCLUDE_DIRS}) + +include_directories( + include + ${CMAKE_SOURCE_DIR}/util/include + ${CMAKE_SOURCE_DIR}/model/include + ${CMAKE_SOURCE_DIR}/parser/include + ${PLUGIN_DIR}/model/include) + +add_library(pythonparser SHARED + src/pythonparser.cpp + src/a/asdf.py) + +find_package (Python) + +target_link_libraries(pythonparser model ${PYTHON_LIBRARIES} ${Boost_LIBRARIES}) + +target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) +target_link_options(pythonparser PUBLIC -Xlinker -export-dynamic) + +install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h new file mode 100644 index 000000000..877b7bab6 --- /dev/null +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -0,0 +1,28 @@ +#ifndef CODECOMPASS_PYTHONPARSER_H +#define CODECOMPASS_PYTHONPARSER_H + + +#include +#include + +namespace cc +{ + namespace parser + { + + class PythonParser : public AbstractParser { + public: + PythonParser(ParserContext &ctx_); + + virtual ~PythonParser(); + + virtual void markModifiedFiles() override; + + virtual bool cleanupDatabase() override; + + virtual bool parse() override; + }; + } // parser +} // cc + +#endif //CODECOMPASS_PYTHONPARSER_H diff --git a/plugins/python/parser/src/my_ast/__init__.py b/plugins/python/parser/src/my_ast/__init__.py new file mode 100644 index 000000000..7d7338820 --- /dev/null +++ b/plugins/python/parser/src/my_ast/__init__.py @@ -0,0 +1,7 @@ +__all__ = ["base_data", "built_in_functions", "built_in_operators", "built_in_types", "class_data", + "class_init_declaration", "class_preprocessor", "file_info", "function_data", "function_symbol_collector", + "function_symbol_collector_factory", "import_finder", "import_preprocessor", "logger", "main", + "member_access_collector", "parser", "parse_exception", "placeholder_function_declaration_cache", + "preprocessed_data", "preprocessed_file", "preprocessed_function", "preprocessed_variable", "python_parser", + "scope", "scope_manager", "symbol_collector", "symbol_collector_interface", "symbol_finder", "type_data", + "type_deduction", "variable_data"] diff --git a/plugins/python/parser/src/my_ast/base_data.py b/plugins/python/parser/src/my_ast/base_data.py new file mode 100644 index 000000000..89c85cb97 --- /dev/null +++ b/plugins/python/parser/src/my_ast/base_data.py @@ -0,0 +1,66 @@ +from typing import List, TypeVar, Generic, Set, Optional + +from my_ast.common.file_position import FilePosition +from my_ast.type_data import DeclarationType + + +class Usage: + def __init__(self, name: str, position: FilePosition): + self.name = name + self.file_position = position + + def __str__(self): + return self.name + ' (' + str(self.file_position) + ')' + + def __repr__(self): + return self.__str__() + + +class Declaration(DeclarationType): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, + declaration_type: Optional[Set[DeclarationType]] = None): + DeclarationType.__init__(self, name, qualified_name) + assert declaration_type is None or not any(isinstance(x, type) for x in declaration_type) + self.file_position: FilePosition = pos + if declaration_type is None: + self.type: Set[DeclarationType] = set() + else: + self.type: Set[DeclarationType] = declaration_type + self.usages: List[Usage] = [] + + def is_same_declaration(self, other: 'Declaration') -> bool: + return self.__eq__(other) and self.file_position == other.file_position + + def is_same_usage(self, other: Usage): + return self.name == other.name and self.file_position == other.file_position + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) + + def get_type_repr(self) -> str: + return '[' + ','.join({x.get_type_repr() if x is not None else 'None' for x in self.type}) + ']' + + +class TypeDeclaration(Declaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition): + super().__init__(name, qualified_name, pos) + self.type.add(self) + + +T = TypeVar('T', bound=Declaration) + + +class ImportedDeclaration(Generic[T]): + def __init__(self, declaration: T): + self.imported_declaration = declaration + + +class DocumentedType: + def __init__(self, documentation: str): + self.documentation: str = documentation diff --git a/plugins/python/parser/src/my_ast/built_in_functions.py b/plugins/python/parser/src/my_ast/built_in_functions.py new file mode 100644 index 000000000..66c46e358 --- /dev/null +++ b/plugins/python/parser/src/my_ast/built_in_functions.py @@ -0,0 +1,612 @@ +import inspect +import sys +from abc import ABC, abstractmethod +from typing import Optional, Set, Union, List + +from my_ast import built_in_types +from my_ast.built_in_types import Boolean, Complex, Dictionary, Float, FrozenSet, Integer, File, Range, Slice, Tuple, \ + Object, MemoryView, String, Bytes, ByteArray, Type +from my_ast.common.file_position import FilePosition +from my_ast.function_data import FunctionDeclaration +from my_ast.type_data import DeclarationType + + +class BuiltInFunction(FunctionDeclaration, ABC): + def __init__(self): + file_position = FilePosition.get_empty_file_position() + qualified_name = "builtins." + self.get_name() + super().__init__(self.get_name(), qualified_name, file_position, [], "", self.get_type()) + if self.get_override() is None: + self.override = [] + else: + self.override: List[str] = self.get_override() + + @staticmethod + @abstractmethod + def get_name() -> str: + pass + + @staticmethod + def get_override() -> Optional[List[str]]: + return None + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return None + + def get_type_repr(self) -> str: + if self.get_type() is None: + return "" + return ','.join([x.get_type_repr() for x in self.get_type()]) + + +class FunctionDecorator(BuiltInFunction, ABC): + pass + + +class AbsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'abs' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__abs__'] + + +class AllFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'all' + + +class AnyFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'any' + + +class AsciiFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'ascii' + + +# returns binary string +class BinFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bin' + + +class BoolFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bool' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class BreakpointFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'breakpoint' + + +class BytearrayFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bytearray' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {ByteArray()} + + +class BytesFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bytes' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Bytes()} + + +class CallableFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'callable' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class ChrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'chr' + + +class ClassmethodFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'classmethod' + + +class CompileFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'compile' + + +class ComplexFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'complex' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__complex__', '__float__', '__index__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Complex()} + + +class DelattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'delattr' + + +class DictFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'dict' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Dictionary()} + + +class DirFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'dir' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__dir__'] + + +class DivmodFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'divmod' + + +# returns enumerate object +class EnumerateFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'enumerate' + + +class EvalFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'eval' + + +class ExecFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'exec' + + +class FilterFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'filter' + + +class FloatFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'float' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Float()} + + +class FormatFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'format' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__format__'] + + +class FrozensetFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'frozenset' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {FrozenSet()} + + +class GetattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'getattr' + + +class GlobalsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'globals' + + +class HasattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hasattr' + + +class HashFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hash' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__hash__'] + + +class HelpFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'help' + + +class HexFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hex' + + +class IdFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'id' + + +class InputFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'input' + + +class IntFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'int' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__init__', '__index__', '__truncate__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Integer()} + + +class IsinstanceFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'isinstance' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class IssubclassFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'issubclass' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +# returns Iterator +class IterFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'iter' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__iter__'] + + +class LenFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'len' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Integer()} + + +class ListFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'list' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {built_in_types.List()} + + +class LocalsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'locals' + + +class MapFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'map' + + +class MaxFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'max' + + +class MemoryViewFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'memoryview' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {MemoryView()} + + +class MinFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'min' + + +class NextFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'next' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__next__'] + + +class ObjectFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'object' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Object()} + + +class OctFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'oct' + + +class OpenFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'open' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {File()} + + +class OrdFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'ord' + + +class PowFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'pow' + + +class PrintFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'print' + + +class PropertyFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'property' + + +class RangeFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'range' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Range()} + + +class ReprFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'repr' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__repr__'] + + +# returns iterator +class ReversedFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'reversed' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__reversed__'] + + +class RoundFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'round' + + +class SetFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'set' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {built_in_types.Set()} + + +class SetattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'setattr' + + +class SliceFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'slice' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Slice()} + + +class SortedFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'sorted' + + +class StaticmethodFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'staticmethod' + + +class StrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'str' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__str__', '__repr__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {String()} + + +class SumFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'sum' + + +class SuperFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'super' + + +class TupleFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'tuple' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Tuple()} + + +class TypeFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'type' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Type()} + + +class VarsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'vars' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__dict__'] + + +class ZipFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'zip' + + +class ImportFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return '__import__' + + +built_in_functions = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltInFunction) and + x is not BuiltInFunction and x is not FunctionDecorator) + + +def get_built_in_function(name: str) -> Optional[BuiltInFunction]: + bif = [x for x in built_in_functions if x[1].get_name() == name] + assert len(bif) <= 1 + if len(bif) == 0: + return None + + return bif[0][1]() diff --git a/plugins/python/parser/src/my_ast/built_in_operators.py b/plugins/python/parser/src/my_ast/built_in_operators.py new file mode 100644 index 000000000..6383a0fb4 --- /dev/null +++ b/plugins/python/parser/src/my_ast/built_in_operators.py @@ -0,0 +1,324 @@ +import ast +import inspect +import sys +from abc import ABC, abstractmethod +from typing import Type, List, Optional + +from my_ast.built_in_types import Complex, Float, Integer, Boolean + + +class BuiltInOperator(ABC): + def __init__(self): + self.ast_type = self.get_ast_type() + self.override = self.get_override() + + @staticmethod + @abstractmethod + def get_ast_type() -> Type[ast.AST]: + pass + + @staticmethod + def get_override() -> List[str]: + return [] + + # @staticmethod + # @abstractmethod + # def get_type() -> Type: + # pass + + +class UnaryOperator(BuiltInOperator, ABC): + pass + + +class BinaryOperator(BuiltInOperator, ABC): + pass + + +class CompareOperator(BuiltInOperator, ABC): + pass + + +class BooleanOperator(BuiltInOperator, ABC): + pass + + +class AddOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Add + + @staticmethod + def get_override() -> List[str]: + return ['__add__'] + + +class SubOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Sub + + @staticmethod + def get_override() -> List[str]: + return ['__sub__'] + + +class MultOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Mult + + @staticmethod + def get_override() -> List[str]: + return ['__mul__'] + + +class MatMultOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.MatMult + + @staticmethod + def get_override() -> List[str]: + return ['__matmul__'] + + +class DivOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Div + + @staticmethod + def get_override() -> List[str]: + return ['__truediv__'] + + +class ModOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Mod + + @staticmethod + def get_override() -> List[str]: + return ['__mod__'] + + +class PowOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Pow + + @staticmethod + def get_override() -> List[str]: + return ['__pow__'] + + +class LShiftOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.LShift + + @staticmethod + def get_override() -> List[str]: + return ['__lshift__'] + + +class RShiftOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.RShift + + @staticmethod + def get_override() -> List[str]: + return ['__rshift__'] + + +class BitOrOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitOr + + @staticmethod + def get_override() -> List[str]: + return ['__or__'] + + +class BitXorOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitXor + + @staticmethod + def get_override() -> List[str]: + return ['__xor__'] + + +class BitAndOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitAnd + + @staticmethod + def get_override() -> List[str]: + return ['__and__'] + + +class FloorDivOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.FloorDiv + + @staticmethod + def get_override() -> List[str]: + return ['__floordiv__'] + + +class AndOperator(BooleanOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.And + + +class OrOperator(BooleanOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Or + + +class EqOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Eq + + @staticmethod + def get_override() -> List[str]: + return ['__eq__'] + + +class NotEqOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.NotEq + + @staticmethod + def get_override() -> List[str]: + return ['__ne__'] + + +class LtOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Lt + + @staticmethod + def get_override() -> List[str]: + return ['__lt__'] + + +class LtEOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.LtE + + @staticmethod + def get_override() -> List[str]: + return ['__le__'] + + +class GtOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Gt + + @staticmethod + def get_override() -> List[str]: + return ['__gt__'] + + +class GtEOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.GtE + + @staticmethod + def get_override() -> List[str]: + return ['__ge__'] + + +class IsOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Is + + +class IsNotOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.IsNot + + +class InOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.In + + +class NotInOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.NotIn + + +class InvertOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Invert + + @staticmethod + def get_override() -> List[str]: + return ['__inv__', '__invert__'] + + +class NotOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Not + + @staticmethod + def get_override() -> List[str]: + return ['__not__'] + + +class UAddOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.UAdd + + @staticmethod + def get_override() -> List[str]: + return ['__pos__'] + + +class USubOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.UAdd + + @staticmethod + def get_override() -> List[str]: + return ['__neg__'] + + +built_in_operators = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltInOperator) and + x is not BuiltInOperator) + +number_precedence = [Complex, Float, Integer, Boolean] + + +def get_built_in_operator(t: ast.AST) -> Optional[BuiltInOperator]: + bio = [x for x in built_in_operators if type(t) is x[1].get_ast_type()] + assert len(bio) <= 1 + if len(bio) == 0: + return None + return bio[0][1]() diff --git a/plugins/python/parser/src/my_ast/built_in_types.py b/plugins/python/parser/src/my_ast/built_in_types.py new file mode 100644 index 000000000..cfbe74256 --- /dev/null +++ b/plugins/python/parser/src/my_ast/built_in_types.py @@ -0,0 +1,174 @@ +import typing +from typing import Optional, Any + +from my_ast.base_data import TypeDeclaration +from my_ast.common.file_position import FilePosition + + +class BuiltIn(TypeDeclaration): + def __init__(self, name: str): + super().__init__(name, "builtins." + name, FilePosition.get_empty_file_position()) + + def get_type_repr(self) -> str: + return self.name + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + +class GenericType: + def __init__(self, types: Optional[typing.Set[Any]] = None): + if types is None: + self.types = set() + else: + self.types = types + + def add_type(self, new_type): + self.types.add(new_type) + + +class GenericBuiltInType(BuiltIn, GenericType): + def __init__(self, name: str, types: Optional[typing.Set[Any]] = None): + BuiltIn.__init__(self, name) + GenericType.__init__(self, types) + + def get_type_repr(self) -> str: + return self.name + '<' + ','.join({x.get_type_repr() for x in self.types}) + '>' + + +class Boolean(BuiltIn): + def __init__(self): + super().__init__('bool') + + +class Integer(BuiltIn): + def __init__(self): + super().__init__('int') + + +class Float(BuiltIn): + def __init__(self): + super().__init__('float') + + +class Complex(BuiltIn): + def __init__(self): + super().__init__('complex') + + +class String(BuiltIn): + def __init__(self): + super().__init__('str') + + +class List(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__('list', types) + + +class Tuple(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__('tuple', types) + + +class Range(BuiltIn): + def __init__(self): + super().__init__('range') + + +class Bytes(BuiltIn): + def __init__(self): + super().__init__('bytes') + + +class ByteArray(BuiltIn): + def __init__(self): + super().__init__('bytearray') + + +class MemoryView(BuiltIn): + def __init__(self): + super().__init__('memoryview') + + +class Set(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__('set', types) + + +class FrozenSet(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__('frozenset', types) + + +class Dictionary(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__('dict', types) + + +class RangeType(BuiltIn): # generic? (implement __index__ method) + def __init__(self): + super().__init__('range') + + +class NoneType(BuiltIn): + def __init__(self): + super().__init__('None') + + +class EllipsisType(BuiltIn): + def __init__(self): + super().__init__('Ellipsis') + + +class Slice(BuiltIn): + def __init__(self): + super().__init__('slice') + + +class Module(BuiltIn): + def __init__(self): + super().__init__('module') + + +class Function(BuiltIn): + def __init__(self): + super().__init__('function') + + +class Method(BuiltIn): + def __init__(self): + super().__init__('method') + + +class Lambda(BuiltIn): + def __init__(self): + super().__init__('lambda') + + +class Generator(BuiltIn): + def __init__(self): + super().__init__('generator') + + +class Type(BuiltIn): + def __init__(self): + super().__init__('type') + + +class Object(BuiltIn): + def __init__(self): + super().__init__('object') + + +class File(BuiltIn): + def __init__(self): + super().__init__('file') + +# others: contextmanager, code (~str?), NotImplemented, internal diff --git a/plugins/python/parser/src/my_ast/class_data.py b/plugins/python/parser/src/my_ast/class_data.py new file mode 100644 index 000000000..1be824f2e --- /dev/null +++ b/plugins/python/parser/src/my_ast/class_data.py @@ -0,0 +1,55 @@ +from typing import List + +from my_ast.common.file_position import FilePosition +from my_ast.base_data import TypeDeclaration, ImportedDeclaration, DocumentedType +import my_ast.function_data as fd +import my_ast.variable_data as vd +from my_ast.persistence.base_dto import UsageDTO + +from my_ast.persistence.class_dto import ClassDeclarationDTO, ClassMembersDTO + + +# TODO: @classmethod +class ClassDeclaration(TypeDeclaration, DocumentedType): + def __init__(self, name: str, qualified_name: str, position: FilePosition, base_classes: List['ClassDeclaration'], + documentation: str): + TypeDeclaration.__init__(self, name, qualified_name, position) + DocumentedType.__init__(self, documentation) + self.base_classes: List[ClassDeclaration] = base_classes + self.methods: List[fd.FunctionDeclaration] = [] + self.static_methods: List[fd.StaticFunctionDeclaration] = [] + self.attributes: List[vd.VariableDeclaration] = [] + self.static_attributes: List[vd.StaticVariableDeclaration] = [] + self.classes: List[ClassDeclaration] = [] + + def create_dto(self) -> ClassDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + base_classes = set() + for base_class in self.base_classes: + base_classes.add(base_class.qualified_name) + members = ClassMembersDTO() + for m in self.methods: + members.methods.append(m.qualified_name) + for sm in self.static_methods: + members.static_methods.append(sm.qualified_name) + for a in self.attributes: + members.attributes.append(a.qualified_name) + for sa in self.static_attributes: + members.static_attributes.append(sa.qualified_name) + for c in self.classes: + members.classes.append(c.qualified_name) + return ClassDeclarationDTO(self.name, self.qualified_name, self.file_position, + types, usages, base_classes, members, self.documentation) + + def get_type_repr(self) -> str: + return self.name + + +class ImportedClassDeclaration(ClassDeclaration, ImportedDeclaration[ClassDeclaration]): + def __init__(self, name: str, pos: FilePosition, class_declaration: ClassDeclaration): + ClassDeclaration.__init__(self, name, "", pos, class_declaration.base_classes, "") + ImportedDeclaration.__init__(self, class_declaration) + self.type = {class_declaration} diff --git a/plugins/python/parser/src/my_ast/class_init_declaration.py b/plugins/python/parser/src/my_ast/class_init_declaration.py new file mode 100644 index 000000000..170e892fd --- /dev/null +++ b/plugins/python/parser/src/my_ast/class_init_declaration.py @@ -0,0 +1,16 @@ +from typing import List, Union, Set, Optional + +import my_ast.base_data as data +from my_ast.built_in_types import BuiltIn +from my_ast.class_data import ClassDeclaration +from my_ast.common.file_position import FilePosition +from my_ast.function_data import FunctionDeclaration, FunctionParameter + + +class ClassInitDeclaration(FunctionDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, params: List[FunctionParameter], + class_declaration: ClassDeclaration, + func_type: Optional[Set[Union[data.Declaration, BuiltIn]]] = None): + super().__init__(name, qualified_name, pos, params, "", + func_type if func_type is not None else {class_declaration}) + self.class_declaration = class_declaration diff --git a/plugins/python/parser/src/my_ast/class_preprocessor.py b/plugins/python/parser/src/my_ast/class_preprocessor.py new file mode 100644 index 000000000..4d2682585 --- /dev/null +++ b/plugins/python/parser/src/my_ast/class_preprocessor.py @@ -0,0 +1,84 @@ +import ast +from typing import List, Dict + +from my_ast.base_data import Declaration +from my_ast.common.utils import has_attr +from my_ast.preprocessed_data import PreprocessedDeclaration +from my_ast.preprocessed_function import PreprocessedFunction +from my_ast.preprocessed_variable import PreprocessedVariable + + +class PreprocessedClass(PreprocessedDeclaration): + def __init__(self, name: str): + super().__init__(name) + self.methods: List[PreprocessedFunction] = [] + self.attributes: List[PreprocessedVariable] = [] + self.classes: List[PreprocessedClass] = [] + self.type_usages: List[Declaration] = [] + + def append_method(self, name: str): + self.methods.append(PreprocessedFunction(name)) + + def append_attribute(self, name: str): + self.attributes.append(PreprocessedVariable(name)) + + def append_class(self, nested_class: 'PreprocessedClass'): # 'type' -> in case of type(self) + self.classes.append(nested_class) + + +class PreprocessedClassCollector(ast.NodeVisitor): + def __init__(self): + self.classes: List[PreprocessedClass] = [] + self.class_list: List[int] = [] + self.class_nest_class_map: Dict[PreprocessedClass, List[ast.ClassDef]] = {} + + def append_class(self, node: ast.ClassDef): + self.class_list.append(len(self.classes)) + preprocessed_class = PreprocessedClass(node.name) + self.classes.append(preprocessed_class) + self.handle_nested_class(node, preprocessed_class) + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.append_method(member) + elif isinstance(member, ast.Assign): + self.append_attribute(member) + elif isinstance(member, ast.ClassDef): + self.append_nested_class(preprocessed_class, member) + elif isinstance(member, ast.Pass): + pass + elif isinstance(member, ast.Expr) and hasattr(member, 'value') and isinstance(member.value, ast.Constant) \ + and hasattr(member.value, "value") and isinstance(member.value.value, str): + pass # TODO: documentation comment + else: + assert False, "Unknown class member: " + str(type(member)) + + def handle_nested_class(self, node: ast.ClassDef, preprocessed_class: PreprocessedClass): + for parent_class in self.class_nest_class_map: + if node in self.class_nest_class_map[parent_class]: + parent_class.append_class(preprocessed_class) + self.class_nest_class_map[parent_class].remove(node) + + def class_processed(self): + del self.class_list[-1] + + def append_method(self, node: ast.FunctionDef): + self.classes[self.class_list[-1]].append_method(node.name) + if node.name == '__init__': + self.visit(node) + + def append_attribute(self, node: ast.Assign): + for attribute in node.targets: + if hasattr(attribute, 'id'): # class variable + self.classes[self.class_list[-1]].append_attribute(attribute.id) + elif hasattr(attribute, 'attr') and has_attr(attribute, 'value.id') and attribute.value.id == 'self': + self.classes[self.class_list[-1]].append_attribute(attribute.attr) + + def append_nested_class(self, node: PreprocessedClass, member: ast.ClassDef): + if node in self.class_nest_class_map: + self.class_nest_class_map[node].append(member) + else: + self.class_nest_class_map[node] = [member] + + def visit_Assign(self, node: ast.Assign): + self.append_attribute(node) + self.generic_visit(node) diff --git a/plugins/python/parser/src/my_ast/common/__init__.py b/plugins/python/parser/src/my_ast/common/__init__.py new file mode 100644 index 000000000..1df5a03e2 --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/__init__.py @@ -0,0 +1 @@ +__all__ = ["hashable_list", "file_position", "parser_tree", "position", "unique_list", "utils"] diff --git a/plugins/python/parser/src/my_ast/common/file_position.py b/plugins/python/parser/src/my_ast/common/file_position.py new file mode 100644 index 000000000..d653e9dd9 --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/file_position.py @@ -0,0 +1,20 @@ +from pathlib import PurePath +from typing import Optional + +from my_ast.common.position import Range + + +class FilePosition: + def __init__(self, file: Optional[PurePath], r: Range): + self.file: Optional[PurePath] = file # Optional -> builtins + self.range: Range = r + + def __str__(self): + return "File: " + str(self.file) + " - " + str(self.range) + + def __repr__(self): + return self.__str__() + + @staticmethod + def get_empty_file_position(): + return FilePosition(None, Range.get_empty_range()) diff --git a/plugins/python/parser/src/my_ast/common/hashable_list.py b/plugins/python/parser/src/my_ast/common/hashable_list.py new file mode 100644 index 000000000..45d6deae5 --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/hashable_list.py @@ -0,0 +1,31 @@ +from collections import Counter +from typing import List, TypeVar, Generic + +T = TypeVar('T') + + +class HashableList(Generic[T], List[T]): + def __hash__(self): + return hash(e for e in self) + + def __eq__(self, other): + return isinstance(other, type(self)) and Counter(self) == Counter(other) + + def __ne__(self, other): + return not self.__eq__(other) + + +class OrderedHashableList(Generic[T], List[T]): + def __hash__(self): + return hash(e for e in self) + + def __eq__(self, other): + if not isinstance(other, type(self)) or len(other) != len(self): + return False + for i in range(0, len(self)): + if self[i] != other[i]: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/my_ast/common/location.py b/plugins/python/parser/src/my_ast/common/location.py new file mode 100644 index 000000000..b6c9a14dd --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/location.py @@ -0,0 +1,8 @@ +from my_ast.common.position import Position + + +class Location: + def __init__(self, path: str, file: str, pos: Position): + self.path = path + self.file = file + self.position = pos diff --git a/plugins/python/parser/src/my_ast/common/parser_tree.py b/plugins/python/parser/src/my_ast/common/parser_tree.py new file mode 100644 index 000000000..cc0d60dab --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/parser_tree.py @@ -0,0 +1,30 @@ +import ast + + +class ParserTreeNode: + def __init__(self, node, parent: 'ParserTreeNode'): + self.node = node + self.parent = parent + self.children = [] + self.process_children() + + def process_children(self): + for child in ast.iter_child_nodes(self.node): + self.children.append(ParserTreeNode(child, self)) + + +class ParserTree: + def __init__(self, node): + self.root = ParserTreeNode(node, None) + + def find_node(self, node) -> ParserTreeNode: + return self.find_node_in_parent(self.root, node) + + def find_node_in_parent(self, parent: ParserTreeNode, node) -> ParserTreeNode: + if parent.node is node: + return parent + for child in parent.children: + n = self.find_node_in_parent(child, node) + if n is not None: + return n + return None diff --git a/plugins/python/parser/src/my_ast/common/position.py b/plugins/python/parser/src/my_ast/common/position.py new file mode 100644 index 000000000..1f7f06187 --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/position.py @@ -0,0 +1,36 @@ +class Position: + def __init__(self, line: int, col: int): + self.line = line + self.column = col + + def __str__(self): + return "line: " + str(self.line) + ", column: " + str(self.column) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.line == other.line and self.column == other.column + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def get_empty_position(): + return Position(0, 0) + + +class Range: + def __init__(self, start_pos: Position, end_pos: Position): + self.start_position = start_pos + self.end_position = end_pos + + def __str__(self): + return "Start position: " + str(self.start_position) + " - End position: " + str(self.end_position) + + def __repr__(self): + return self.__str__() + + @staticmethod + def get_empty_range(): + return Range(Position.get_empty_position(), Position.get_empty_position()) diff --git a/plugins/python/parser/src/my_ast/common/unique_list.py b/plugins/python/parser/src/my_ast/common/unique_list.py new file mode 100644 index 000000000..952f707ae --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/unique_list.py @@ -0,0 +1,23 @@ +from typing import TypeVar, List, Iterable + +T = TypeVar('T') + + +class UniqueList(List[T]): + def __init__(self, seq=()): + super().__init__() + self.extend(seq) + + def append(self, obj: T) -> None: + if obj in self: + self.remove(obj) + super().append(obj) + + def extend(self, iterable: Iterable[T]) -> None: + for i in iterable: + self.append(i) + + def insert(self, index: int, obj: T) -> None: + if obj in self: + self.remove(obj) + super().insert(index, obj) diff --git a/plugins/python/parser/src/my_ast/common/utils.py b/plugins/python/parser/src/my_ast/common/utils.py new file mode 100644 index 000000000..db3519ea8 --- /dev/null +++ b/plugins/python/parser/src/my_ast/common/utils.py @@ -0,0 +1,18 @@ +import ast + +from my_ast.common.position import Range, Position + + +def has_attr(obj, attrs) -> bool: + for attr in attrs.split("."): + if hasattr(obj, attr): + obj = getattr(obj, attr) + else: + return False + return True + + +def create_range_from_ast_node(node: ast.AST) -> Range: + start_pos = Position(node.lineno, node.col_offset) + end_pos = Position(node.end_lineno, node.end_col_offset) + return Range(start_pos, end_pos) diff --git a/plugins/python/parser/src/my_ast/db_test.log b/plugins/python/parser/src/my_ast/db_test.log new file mode 100644 index 000000000..9976880ff --- /dev/null +++ b/plugins/python/parser/src/my_ast/db_test.log @@ -0,0 +1,27 @@ + +FILE: __init__ +========================================= +Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] + +FILE: asdf +========================================= + +FILE: qwer +========================================= +Func: g (line: 1, column: 0, length: 1) {global - } [] + +FILE: asdf +========================================= +Var: a (line: 4, column: 0, length: 1) {function - f} [Placeholder] +Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] +Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] +Var: b (line: 4, column: 0, length: 1) {function - f} [Placeholder] +Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] +Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] +Func: g (line: 0, column: 0, length: 0) {global - } [] +Func Usage: g (line: 7, column: 4, length: 1) [] +Func: f (line: 4, column: 0, length: 1) {global - } [str] + +FILE: __init__ +========================================= +Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] diff --git a/plugins/python/parser/src/my_ast/file_info.py b/plugins/python/parser/src/my_ast/file_info.py new file mode 100644 index 000000000..a79588140 --- /dev/null +++ b/plugins/python/parser/src/my_ast/file_info.py @@ -0,0 +1,59 @@ +from enum import Enum, unique, auto +from pathlib import PurePath, Path +from typing import Optional + +from my_ast.persistence.file_content_dto import FileContentDTO +from my_ast.persistence.file_dto import FileDTO +from my_ast.preprocessed_file import PreprocessedFile +from my_ast.symbol_collector import SymbolCollector + + +@unique +class ProcessStatus(Enum): + WAITING = auto() + PREPROCESSED = auto() + PROCESSED = auto() + + +# TODO: expr_1; import; expr_2; - correct script -> not a problem +class FileInfo: + def __init__(self, file: str, path: Optional[PurePath]): # TODO: remove optional + self.file: str = file + self.path: Optional[PurePath] = path + self.symbol_collector: Optional[SymbolCollector] = None + self.preprocessed_file: PreprocessedFile = PreprocessedFile() + self.status: ProcessStatus = ProcessStatus.WAITING + + def preprocess_file(self, tree) -> None: + self.preprocessed_file.visit(tree) + self.status = ProcessStatus.PREPROCESSED + + def set_variable_collector(self, variable_collector: SymbolCollector) -> None: + self.symbol_collector = variable_collector + self.status = ProcessStatus.PROCESSED + + def create_dto(self) -> FileDTO: + file_dto = FileDTO() + file_dto.path = str(self.path) + file_dto.file_name = self.file + file_dto.timestamp = Path(self.path).stat().st_mtime + file_dto.content = FileContentDTO(self.get_content(self.path)) + file_dto.parent = str(self.path.parent) + file_dto.parse_status = self.get_parse_status(self.status) + file_dto.documentation = self.preprocessed_file.documentation + return file_dto + + @staticmethod + def get_content(file: PurePath) -> str: + with open(file) as f: + return f.read() + + @staticmethod + def get_parse_status(status: ProcessStatus) -> int: + if status == ProcessStatus.WAITING: + return 0 + elif status == ProcessStatus.PREPROCESSED: + return 1 + elif status == ProcessStatus.PROCESSED: + return 2 + assert False diff --git a/plugins/python/parser/src/my_ast/function_data.py b/plugins/python/parser/src/my_ast/function_data.py new file mode 100644 index 000000000..21a9ec5ad --- /dev/null +++ b/plugins/python/parser/src/my_ast/function_data.py @@ -0,0 +1,41 @@ +from typing import List, Optional, Set, Union + +from my_ast.common.file_position import FilePosition +import my_ast.base_data as data +from my_ast.persistence.base_dto import UsageDTO +from my_ast.persistence.function_dto import FunctionDeclarationDTO +from my_ast.type_data import DeclarationType + + +class FunctionParameter: + def __init__(self, name: str, func_type: str = None): + self.name: str = name + self.type: Optional[str] = func_type + + +class FunctionDeclaration(data.Declaration, data.DocumentedType): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, params: List[FunctionParameter], + documentation: str, func_type: Optional[Set[Union[DeclarationType]]] = None): + data.Declaration.__init__(self, name, qualified_name, pos, func_type) + data.DocumentedType.__init__(self, documentation) + self.parameters: List[FunctionParameter] = params + + def create_dto(self) -> FunctionDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + for t in self.type: + types.add(t.qualified_name) + return FunctionDeclarationDTO(self.name, self.qualified_name, self.file_position, + types, usages, self.documentation) + + +class StaticFunctionDeclaration(FunctionDeclaration): + pass + + +class ImportedFunctionDeclaration(FunctionDeclaration, data.ImportedDeclaration[FunctionDeclaration]): + def __init__(self, name: str, pos: FilePosition, func_declaration: FunctionDeclaration): + FunctionDeclaration.__init__(self, name, "", pos, func_declaration.parameters, "", func_declaration.type) + data.ImportedDeclaration.__init__(self, func_declaration) diff --git a/plugins/python/parser/src/my_ast/function_symbol_collector.py b/plugins/python/parser/src/my_ast/function_symbol_collector.py new file mode 100644 index 000000000..2bbe13519 --- /dev/null +++ b/plugins/python/parser/src/my_ast/function_symbol_collector.py @@ -0,0 +1,64 @@ +import ast +from typing import Union, Any, List + +from my_ast.common.file_position import FilePosition +from my_ast.common.parser_tree import ParserTree +from my_ast.common.utils import create_range_from_ast_node +from my_ast.function_data import FunctionDeclaration +from my_ast.scope import FunctionScope +from my_ast.symbol_collector import SymbolCollector +from my_ast.symbol_collector_interface import IFunctionSymbolCollector +from my_ast.type_data import DeclarationType +from my_ast.type_deduction import TypeDeduction +from my_ast.variable_data import VariableDeclaration + + +class TemporaryScope(FunctionScope): + pass + + +class FunctionSymbolCollector(SymbolCollector, IFunctionSymbolCollector): + def __init__(self, symbol_collector: SymbolCollector, tree: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[DeclarationType]): + SymbolCollector.__init__(self, ParserTree(tree), symbol_collector.current_file, + symbol_collector.preprocessed_file, + symbol_collector.import_finder, + symbol_collector.function_symbol_collector_factory) + IFunctionSymbolCollector.__init__(self) + self.scope_manager = symbol_collector.scope_manager + self.type_deduction = TypeDeduction(self, self.scope_manager, self.function_symbol_collector_factory) + self.arguments = arguments + + # TODO: handle local functions + # TODO: circular import + placeholder? + def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: + r = create_range_from_ast_node(node) + self.function = FunctionDeclaration(node.name, + self.scope_manager.get_qualified_name_from_current_scope(node.name), + FilePosition(self.current_file, r), [], "") + # self.scope_manager.append_function_to_current_scope(self.function) + self.current_function_declaration.append(self.function) + + def action(): + self.collect_parameters(node) + self.generic_visit(node) + + self.scope_manager.with_scope(FunctionScope(node.name), action) + del self.current_function_declaration[-1] + + def collect_parameters(self, node: ast.FunctionDef) -> None: + arg_idx = 0 + for param in node.args.args: + if hasattr(param, 'arg'): # ast.Name + r = create_range_from_ast_node(node) + new_variable = VariableDeclaration(param.arg, + self.scope_manager.get_qualified_name_from_current_scope(param.arg), + FilePosition(self.current_file, r)) + if param.arg == 'self' and self.scope_manager.is_current_scope_method() and node.args.args[0] is param: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + else: + new_variable.type.update(self.type_deduction.get_current_type({self.arguments[arg_idx]})) + arg_idx += 1 + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + assert False, "Parameter with unknown variable name" diff --git a/plugins/python/parser/src/my_ast/function_symbol_collector_factory.py b/plugins/python/parser/src/my_ast/function_symbol_collector_factory.py new file mode 100644 index 000000000..88d5234f6 --- /dev/null +++ b/plugins/python/parser/src/my_ast/function_symbol_collector_factory.py @@ -0,0 +1,14 @@ +import ast +from abc import ABC, abstractmethod +from typing import Union, List + +from my_ast.symbol_collector_interface import SymbolCollectorBase, IFunctionSymbolCollector +from my_ast.type_data import DeclarationType + + +class FunctionSymbolCollectorFactory(ABC): + @abstractmethod + def get_function_symbol_collector(self, symbol_collector: SymbolCollectorBase, + func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[DeclarationType]) -> IFunctionSymbolCollector: + pass diff --git a/plugins/python/parser/src/my_ast/import_finder.py b/plugins/python/parser/src/my_ast/import_finder.py new file mode 100644 index 000000000..93eb5238e --- /dev/null +++ b/plugins/python/parser/src/my_ast/import_finder.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from pathlib import PurePath +from typing import Optional + + +class ImportFinder(ABC): + # Optional[FileInfo] - circular import + @abstractmethod + def get_file_by_location(self, location: PurePath) -> Optional: + pass + + # Optional[GlobalScope] - circular import + def get_global_scope_by_location(self, location: PurePath) -> Optional: + file = self.get_file_by_location(location) + if file is not None and file.symbol_collector is not None: + assert file.symbol_collector.scope_manager.get_size() == 1 + return file.symbol_collector.scope_manager.get_global_scope() + elif file is not None: + return file.preprocessed_file + return None + + def get_scope_manager_by_location(self, location: PurePath) -> Optional: + file = self.get_file_by_location(location) + if file is not None and file.symbol_collector is not None: + assert file.symbol_collector.scope_manager.get_size() == 1 + return file.symbol_collector.scope_manager + return None diff --git a/plugins/python/parser/src/my_ast/import_preprocessor.py b/plugins/python/parser/src/my_ast/import_preprocessor.py new file mode 100644 index 000000000..1bbdecf26 --- /dev/null +++ b/plugins/python/parser/src/my_ast/import_preprocessor.py @@ -0,0 +1,129 @@ +import ast +from pathlib import PurePath +from typing import Optional, List +from importlib import util # python 3.4 or above + +from my_ast.common.position import Range +from my_ast.common.utils import create_range_from_ast_node + + +class FileDependency: + def __init__(self, location: PurePath, module: str, alias: str = None): + self.location = location + self.module: str = module + self.alias: Optional[str] = alias + + def get_file(self) -> str: + module_parts = self.module.split('.') + return module_parts[-1] + '.py' + + +class ModuleImport: + def __init__(self, name: str, alias: Optional[str] = None): + self.name: str = name + self.alias: Optional[str] = alias + + +class SymbolImport: + def __init__(self, name: str, alias: Optional[str] = None): + self.name: str = name + self.alias: Optional[str] = alias + + def is_all_imported(self) -> bool: + return self.name == '*' + + +class Import: + def __init__(self, path: List[str], module: ModuleImport, imported: List[SymbolImport], + location: Optional[PurePath], r: Range, is_local: bool = False): + self.path: List[str] = path + self.module: ModuleImport = module + self.imported: List[SymbolImport] = imported + self.location: Optional[PurePath] = location + self.range = r + self.is_local = is_local + + def is_module_import(self): + return len(self.imported) == 0 + + +# TODO: need locations of files, and check if modules are among them? +# TODO: handle relative import: +# '.': from the package (parent directory) of the current module (from .'package' import 'module') +# from the __init__ (from . import 'symbol') +# '..': from the grandparent directory of the current module (and so on) +class ImportTable: + def __init__(self): + self.modules: List[Import] = [] + + def append_import(self, node: ast.Import, is_local: bool = False): + self.modules.extend(self.convert_ast_import_to_import(node, is_local)) + + def append_import_from(self, node: ast.ImportFrom, is_local: bool = False): + self.modules.extend(self.convert_ast_import_from_to_import(node, is_local)) + + @staticmethod + def convert_ast_import_to_import(node: ast.Import, is_local: bool = False) -> List[Import]: + imports = [] + for module in node.names: + module_parts = module.name.split('.') + module_name = module_parts[-1] + module_parts.remove(module_name) + r = create_range_from_ast_node(node) + imports.append(ImportTable.create_import(module_parts, module_name, module.asname, [], r, is_local)) + return imports + + @staticmethod + def convert_ast_import_from_to_import(node: ast.ImportFrom, is_local: bool = False) -> List[Import]: + assert len(node.names) > 0 + is_module = True + try: + if util.find_spec(node.module + '.' + node.names[0].name) is None: + is_module = False + except (AttributeError, ModuleNotFoundError): # v3.7: before: AttributeError, after: ModuleNotFoundError + is_module = False + imports = [] + r = create_range_from_ast_node(node) + if is_module: # modules + for module in node.names: + imports.append( + ImportTable.create_import(node.module.split('.'), module.name, module.asname, [], r, is_local)) + else: # variables, functions + imported = [] + for module in node.names: + imported.append(SymbolImport(module.name, module.asname)) + module_parts = node.module.split('.') + module_name = module_parts[-1] + module_parts.remove(module_name) + imports.append(ImportTable.create_import(module_parts, module_name, None, imported, r, is_local)) + return imports + + @staticmethod + def create_import(path: List[str], name: str, alias: Optional[str], imported: List[SymbolImport], + r: Range, is_local: bool = False) -> Import: + location = None + module = path + module.append(name) + try: + spec = util.find_spec('.'.join(module)) + if spec is None: + assert False, "Cannot find module: " + '.'.join(module) + elif spec.has_location: + location = spec.origin + except (AttributeError, ModuleNotFoundError): # v3.7: before: AttributeError, after: ModuleNotFoundError + assert False, "Cannot find module: " + '.'.join(module) + optional_location = None + if location is not None: + optional_location = PurePath(location) + return Import(path, ModuleImport(name, alias), imported, optional_location, r, is_local) + + def get_dependencies(self) -> List[FileDependency]: + dependencies = [] + for module in self.modules: + if len(module.path) == 0: + dependencies.append(FileDependency(module.location, module.module.name, module.module.alias)) + else: + dependencies.append( + FileDependency( + module.location, '.'.join(module.path) + '.' + module.module.name, module.module.alias)) + return dependencies diff --git a/plugins/python/parser/src/my_ast/logger.py b/plugins/python/parser/src/my_ast/logger.py new file mode 100644 index 000000000..a4621e7e2 --- /dev/null +++ b/plugins/python/parser/src/my_ast/logger.py @@ -0,0 +1,16 @@ +import pathlib +import logging + +directory = pathlib.Path(__file__).resolve().parent + +logger = logging.getLogger('test') +logger.setLevel(logging.DEBUG) +file_handler = logging.FileHandler(str(directory) + '/test.log', 'w') +file_handler.setLevel(logging.DEBUG) +logger.addHandler(file_handler) + +db_logger = logging.getLogger('db_test') +db_logger.setLevel(logging.DEBUG) +db_file_handler = logging.FileHandler(str(directory) + '/db_test.log', 'w') +db_file_handler.setLevel(logging.DEBUG) +db_logger.addHandler(db_file_handler) diff --git a/plugins/python/parser/src/my_ast/main.py b/plugins/python/parser/src/my_ast/main.py new file mode 100644 index 000000000..6ebef57a1 --- /dev/null +++ b/plugins/python/parser/src/my_ast/main.py @@ -0,0 +1,43 @@ +import ast +import os +from pathlib import PurePath +from typing import Optional + +from my_ast.common.parser_tree import ParserTree +from my_ast.file_info import FileInfo +from my_ast.import_finder import ImportFinder +from my_ast.parse_exception import ParseException +from my_ast.parser import Parser +from my_ast.symbol_collector import SymbolCollector + + +def main(): + # with open("../test_dir/test_file_2.py", "r") as source: + # tree = ast.parse(source.read()) + # + # pt = ParserTree(tree) + # + # fi = FileInfo("test_file_2.py", PurePath("../test_dir/test_file_2.py")) + # + # class ImportFinderHelper(ImportFinder): + # def get_file_by_location(self, location: PurePath) -> Optional: + # return None + # + # vdc = SymbolCollector(pt, fi.preprocessed_file, ImportFinderHelper()) + # vdc.collect_symbols() + + def directory_exception(path: PurePath) -> bool: + directory = os.path.basename(os.path.normpath(str(path))) + return directory.startswith('.') or directory == 'venv' + + def file_exception(path: PurePath) -> bool: + return False + + exception = ParseException(directory_exception, file_exception) + p = Parser(["F:/ELTE/PythonPlugin"], exception) + p.parse() + pass + + +if __name__ == '__main__': + main() diff --git a/plugins/python/parser/src/my_ast/member_access_collector.py b/plugins/python/parser/src/my_ast/member_access_collector.py new file mode 100644 index 000000000..92a9fcbc9 --- /dev/null +++ b/plugins/python/parser/src/my_ast/member_access_collector.py @@ -0,0 +1,210 @@ +import ast +from typing import Optional, List, Any, Union + + +class MemberAccessCollector(ast.NodeVisitor): + class MemberData: + def __init__(self, name: Optional[str]): + self.name: str = name + + class ConstantData(MemberData): + pass + + class AttributeData(MemberData): + pass + + class MethodData(MemberData): + def __init__(self, name: Optional[str], args): + super().__init__(name) + self.arguments = args + + class ReturnValueData(MemberData): + def __init__(self, name: Optional[str], member_data, node: ast.AST): + super().__init__(name) + self.return_of: MemberAccessCollector.MemberData = member_data + self.node = node + + class SubscriptData(MemberData): + class Index: + def __init__(self, idx): + self.idx = idx + + class Slice: + def __init__(self, lower, upper, step): + self.lower = lower + self.upper = upper + self.step = step + + def __init__(self, name: Optional[str], sub_slice: List[Union[Index, Slice]]): + super().__init__(name) + self.slice: List[Union[MemberAccessCollector.SubscriptData.Index, + MemberAccessCollector.SubscriptData.Slice]] = sub_slice + + class OperatorData(MemberData): + def __init__(self, name: Optional[str], node: Union[ast.BoolOp, ast.BinOp, ast.UnaryOp, ast.Compare]): + super().__init__(name) + + class LambdaData(MemberData): + def __init__(self, name: Optional[str], node: ast.Lambda): + super().__init__(name) + + def __init__(self, member: Union[ast.Call, ast.Attribute]): + self.call_list: List[MemberAccessCollector.MemberData] = [] + self.arguments = [] + self.last = False + self.visit(member) + + def generic_visit(self, node: ast.AST): + if self.last: + return + if not isinstance(node, ast.Await): # await must be skipped, and process the callable + print('Unhandled node type: ' + str(type(node))) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Call(self, node: ast.Call) -> Any: + if self.last: + return + + def process_call(value: ast.AST): + if isinstance(value, (ast.Name, ast.Attribute, ast.Lambda)): + self.call_list.append(self.MethodData(None, self.collect_arguments(node))) + elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): + self.call_list.append( + self.ReturnValueData(None, self.MethodData(None, self.collect_arguments(node)), value)) + elif isinstance(value, ast.Await): + process_call(value.value) + else: + assert False + process_call(node.func) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Subscript(self, node: ast.Subscript) -> Any: + if self.last: + return + + def process_subscript(value: ast.AST): + if isinstance(value, (ast.Name, ast.Attribute)): + self.call_list.append(self.SubscriptData(None, self.collect_slice(node))) + elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): + self.call_list.append( + self.ReturnValueData(None, self.SubscriptData(None, self.collect_slice(node)), value)) + elif isinstance(value, ast.Await): + process_subscript(value.value) + else: + assert False, "Not subscriptable type: " + str(type(value)) + process_subscript(node.value) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Attribute(self, node: ast.Attribute) -> Any: + if self.last: + return + if len(self.call_list) > 0 and self.call_list[-1].name is None: + self.call_list[-1].name = node.attr + else: + self.call_list.append(self.AttributeData(node.attr)) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Name(self, node: ast.Name) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = node.id + else: + self.call_list.append(self.AttributeData(node.id)) + self.last = True + + def visit_Constant(self, node: ast.Constant) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = node.value + else: + value = node.value + if isinstance(value, str): + value = '"' + value + '"' + self.call_list.append(self.ConstantData(value)) + self.last = True + + def visit_Lambda(self, node: ast.Lambda) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = 'lambda' + else: + self.call_list.append(self.LambdaData('lambda', node)) + self.last = True + + def visit_BinOp(self, node: ast.BinOp) -> Any: + self.last = True + + def visit_UnaryOp(self, node: ast.UnaryOp) -> Any: + self.last = True + + def visit_BoolOp(self, node: ast.BoolOp) -> Any: + self.last = True + + def visit_Compare(self, node: ast.Compare) -> Any: + self.last = True + + # TODO: is it necessary? + @staticmethod + def collect_arguments(call: ast.Call) -> List: + arguments = [] + for arg in call.args: + arguments.append(arg) + return arguments + + @staticmethod + def collect_slice(subscript: ast.Subscript) -> List[Union[SubscriptData.Index, SubscriptData.Slice]]: + # TODO: in python 3.9 this must be changed (ast.Index, ast.ExtSlice are deprecated) + sub_slice = [] + + def process_slice(node: (ast.Index, ast.Slice, ast.ExtSlice)): + if isinstance(node, ast.Index): + sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) + elif isinstance(node, ast.Slice): + sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) + elif isinstance(node, ast.ExtSlice): # 3.9 -> ast.Tuple + for dim in node.dims: + process_slice(dim) + else: + assert False, "Unknown slice: " + str(type(node)) + + process_slice(subscript.slice) + return sub_slice + + @staticmethod + def is_callable_after_override(node: ast.AST) -> bool: + return isinstance(node, ast.BoolOp) or isinstance(node, ast.BinOp) or isinstance(node, ast.UnaryOp) or \ + isinstance(node, ast.Compare) + + +class MemberAccessCollectorIterator: + def __init__(self, member_access_collector: MemberAccessCollector): + self.mac = member_access_collector + self.index = len(self.mac.call_list) + + def __iter__(self): + self.index = len(self.mac.call_list) + return self + + def __next__(self): + if self.index <= 0: + raise StopIteration + self.index -= 1 + return self.mac.call_list[self.index] + + def get_current(self): + return self.mac.call_list[self.index] + + def get_first(self): + return self.mac.call_list[-1] + + def get_last(self): + return self.mac.call_list[0] + + def is_iteration_over(self) -> bool: + return self.index <= 0 + + def is_iteration_started(self) -> bool: + return self.index < len(self.mac.call_list) diff --git a/plugins/python/parser/src/my_ast/parse_exception.py b/plugins/python/parser/src/my_ast/parse_exception.py new file mode 100644 index 000000000..2ff8a6448 --- /dev/null +++ b/plugins/python/parser/src/my_ast/parse_exception.py @@ -0,0 +1,20 @@ +from pathlib import PurePath +from typing import Callable # python 3.5 or above + + +class ParseException: + def __init__(self, is_dictionary_exception: Callable[[PurePath], bool], + is_file_exception: Callable[[PurePath], bool]): + self.is_dictionary_exception = is_dictionary_exception + self.is_file_exception = is_file_exception + + def is_dictionary_exception(self, dictionary: PurePath) -> bool: + return self.is_dictionary_exception(dictionary) + + def is_file_exception(self, file: PurePath) -> bool: + return self.is_file_exception(file) + + +class DefaultParseException(ParseException): + def __init__(self): + super().__init__(lambda x: False, lambda x: False) diff --git a/plugins/python/parser/src/my_ast/parser.py b/plugins/python/parser/src/my_ast/parser.py new file mode 100644 index 000000000..8aa8472c8 --- /dev/null +++ b/plugins/python/parser/src/my_ast/parser.py @@ -0,0 +1,127 @@ +import ast +import os +import sys +from pathlib import PurePath +from typing import List, Optional, Union + +from my_ast.common.parser_tree import ParserTree +from my_ast.file_info import FileInfo, ProcessStatus +from my_ast.function_symbol_collector import FunctionSymbolCollector +from my_ast.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from my_ast.import_finder import ImportFinder +from my_ast.logger import logger, db_logger +from my_ast.parse_exception import DefaultParseException, ParseException +from my_ast.persistence.persistence import model_persistence +from my_ast.scope import GlobalScope +from my_ast.symbol_collector import SymbolCollector + +# TODO: builtins - dir(builtins) +# TODO: change logging to DB storing +from my_ast.symbol_collector_interface import IFunctionSymbolCollector +from my_ast.type_data import DeclarationType + +current_file = "" # TODO: only for debug, remove it + + +class Parser(ast.NodeVisitor, ImportFinder, FunctionSymbolCollectorFactory): + def __init__(self, directories: List[str], exceptions: Optional[ParseException] = None): + self.directories: List[str] = directories + if exceptions is None: + exceptions = DefaultParseException() + self.exceptions: ParseException = exceptions + self.files: List[FileInfo] = [] + self.parsing_started_files = [] + self.collect_files() + + def collect_files(self): + for directory in self.directories: + sys.path.append(directory) + self.process_directory(PurePath(directory)) + + def process_directory(self, directory: PurePath): + sub_directories = [] + for element in os.listdir(str(directory)): + path = directory.joinpath(element) + if os.path.isfile(str(path)): + if self.exceptions.is_file_exception(path): + continue + if element.endswith(".py"): + self.files.append(FileInfo(element, path)) + elif os.path.isdir(str(path)): + if self.exceptions.is_dictionary_exception(path): + continue + sub_directories.append(path) + else: + assert False, "Unknown element (file, directory): " + str(path) + + for subdir in sub_directories: + self.process_directory(subdir) + + def parse(self) -> None: + for file_info in self.files: + if file_info.symbol_collector is None: + self.parse_file(file_info) + self.persist_global_scopes() + + def parse_file(self, file_info: FileInfo) -> None: + global current_file + current_file = file_info.file + if file_info.status != ProcessStatus.WAITING: + return + # if file_info in self.parsing_started_files: + # return + # self.parsing_started_files.append(file_info) + logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + db_logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + # TODO: fix this... + # if file_info.file[:-3] in builtin_module_names: # [:-3] - '.py' extension + # return # TODO: handle this case (+ deprecated modules!) + if file_info.path is None: + return + try: + with open(str(file_info.path), "r") as source: + tree = ast.parse(source.read()) + file_info.preprocess_file(tree) + for dependency in file_info.preprocessed_file.import_table.get_dependencies(): + dependency_file_info = [x for x in self.files if x.path == dependency.location] + if len(dependency_file_info) == 0: + pass + # TODO: do we need this? (eg. PYTHONPATH) + # new_file_info = FileInfo(file, dependency.location) + # self.files.append(new_file_info) + # self.parse_file(new_file_info) + # logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + elif len(dependency_file_info) == 1: + if dependency_file_info[0].status is ProcessStatus.WAITING: + self.parse_file(dependency_file_info[0]) + current_file = file_info.file + logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + db_logger. \ + debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + else: + assert False, 'Multiple file occurrence: ' + dependency.get_file() + sc = SymbolCollector(ParserTree(tree), file_info.path, file_info.preprocessed_file, self, self) + sc.collect_symbols() + file_info.set_variable_collector(sc) + + if model_persistence is not None: + model_persistence.persist_file(file_info.create_dto()) + except FileNotFoundError: + return + + def get_file_by_location(self, location: PurePath) -> Optional[FileInfo]: + for file in self.files: + if file.path == location: + return file + return None + + def persist_global_scopes(self): + for file_info in self.files: + if file_info.symbol_collector is not None: + assert isinstance(file_info.symbol_collector.scope_manager.get_current_scope(), GlobalScope) + file_info.symbol_collector.scope_manager.persist_current_scope() + + def get_function_symbol_collector(self, symbol_collector: SymbolCollector, + func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[DeclarationType]) -> IFunctionSymbolCollector: + return FunctionSymbolCollector(symbol_collector, func_def, arguments) diff --git a/plugins/python/parser/src/my_ast/persistence/__init__.py b/plugins/python/parser/src/my_ast/persistence/__init__.py new file mode 100644 index 000000000..f15edfb7d --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/__init__.py @@ -0,0 +1,2 @@ +__all__ = ["build_action", "build_source_target", "file_dto", "file_content_dto", "persistence", "class_dto", + "function_dto", "import_dto.py", "variable_dto", "base_dto"] diff --git a/plugins/python/parser/src/my_ast/persistence/base_dto.py b/plugins/python/parser/src/my_ast/persistence/base_dto.py new file mode 100644 index 000000000..54a03e048 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/base_dto.py @@ -0,0 +1,18 @@ +from typing import List, Set + +from my_ast.common.file_position import FilePosition + + +class UsageDTO: + def __init__(self, file_position: FilePosition): + self.file_position: FilePosition = file_position + + +class DeclarationDTO: + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO]): + self.name: str = name + self.qualified_name: str = qualified_name + self.file_position: FilePosition = file_position + self.type: Set[str] = types # qualified names + self.usages: List[UsageDTO] = usages diff --git a/plugins/python/parser/src/my_ast/persistence/build_action.py b/plugins/python/parser/src/my_ast/persistence/build_action.py new file mode 100644 index 000000000..55e3575ff --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/build_action.py @@ -0,0 +1,11 @@ +from typing import List + +import my_ast.persistence.build_source_target as bst + + +class BuildAction: + def __init__(self, command: str, sources: List[bst.BuildSource], targets: List[bst.BuildTarget]): + self.command = command + self.type = 0 # compile + self.build_sources: List[bst.BuildSource] = sources + self.build_target: List[bst.BuildTarget] = targets diff --git a/plugins/python/parser/src/my_ast/persistence/build_source_target.py b/plugins/python/parser/src/my_ast/persistence/build_source_target.py new file mode 100644 index 000000000..48cc2a131 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/build_source_target.py @@ -0,0 +1,14 @@ +import my_ast.persistence.build_action as ba +from my_ast.persistence.file_dto import FileDTO + + +class BuildSource: + def __init__(self, file: FileDTO, action: ba.BuildAction): + self.file: FileDTO = file + self.action: ba.BuildAction = action + + +class BuildTarget: + def __init__(self, file: FileDTO, action: ba.BuildAction): + self.file: FileDTO = file + self.action: ba.BuildAction = action diff --git a/plugins/python/parser/src/my_ast/persistence/class_dto.py b/plugins/python/parser/src/my_ast/persistence/class_dto.py new file mode 100644 index 000000000..67922ad1c --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/class_dto.py @@ -0,0 +1,23 @@ +from typing import Set, List + +from my_ast.common.file_position import FilePosition +from my_ast.persistence.base_dto import UsageDTO, DeclarationDTO +from my_ast.persistence.documented_dto import DocumentedDTO + + +class ClassMembersDTO: + def __init__(self): + self.methods: List[str] = [] + self.static_methods: List[str] = [] + self.attributes: List[str] = [] + self.static_attributes: List[str] = [] + self.classes: List[str] = [] + + +class ClassDeclarationDTO(DeclarationDTO, DocumentedDTO): + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO], base_classes: Set[str], members: ClassMembersDTO, documentation: str): + DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) + DocumentedDTO.__init__(self, documentation) + self.base_classes: Set[str] = base_classes + self.members: ClassMembersDTO = members diff --git a/plugins/python/parser/src/my_ast/persistence/documented_dto.py b/plugins/python/parser/src/my_ast/persistence/documented_dto.py new file mode 100644 index 000000000..271e09b67 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/documented_dto.py @@ -0,0 +1,3 @@ +class DocumentedDTO: + def __init__(self, documentation: str): + self.documentation: str = documentation diff --git a/plugins/python/parser/src/my_ast/persistence/file.py b/plugins/python/parser/src/my_ast/persistence/file.py new file mode 100644 index 000000000..f4d833539 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/file.py @@ -0,0 +1,38 @@ +from enum import unique, Enum +from pathlib import PurePath, Path +from my_ast.file_info import ProcessStatus, FileInfo +from my_ast.persistence.file_content import FileContent + + +@unique +class ParseStatus(Enum): + PSNONE = 0 + PSPARTIALLYPARSED = 1 + PSFULLYPARSED = 2 + + +class File: + def __init__(self, file_info: FileInfo): + # id? + self.type: str = 'Python' # dir, unknown, binary? + self.path: PurePath = file_info.path + self.file_name: str = file_info.file + self.timestamp = Path(file_info.path).stat().st_mtime # st_birthtime + self.content: FileContent = FileContent(self.get_content(file_info.path)) + self.parent: PurePath = file_info.path.parent + self.parse_status: ParseStatus = self.get_parse_status(file_info.status) + + @staticmethod + def get_content(file: PurePath) -> str: + with open(file) as f: + return f.read() + + @staticmethod + def get_parse_status(status: ProcessStatus) -> ParseStatus: + if status == ProcessStatus.WAITING: + return ParseStatus.PSNONE + elif status == ProcessStatus.PREPROCESSED: + return ParseStatus.PSPARTIALLYPARSED + elif status == ProcessStatus.PROCESSED: + return ParseStatus.PSFULLYPARSED + assert False diff --git a/plugins/python/parser/src/my_ast/persistence/file_content.py b/plugins/python/parser/src/my_ast/persistence/file_content.py new file mode 100644 index 000000000..2e7baf9ca --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/file_content.py @@ -0,0 +1,3 @@ +class FileContent: + def __init__(self, content: str): + self.content: str = content diff --git a/plugins/python/parser/src/my_ast/persistence/file_content_dto.py b/plugins/python/parser/src/my_ast/persistence/file_content_dto.py new file mode 100644 index 000000000..1487d9259 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/file_content_dto.py @@ -0,0 +1,3 @@ +class FileContentDTO: + def __init__(self, content: str): + self.content: str = content diff --git a/plugins/python/parser/src/my_ast/persistence/file_dto.py b/plugins/python/parser/src/my_ast/persistence/file_dto.py new file mode 100644 index 000000000..d66192340 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/file_dto.py @@ -0,0 +1,23 @@ +from enum import unique, Enum + +from my_ast.persistence.documented_dto import DocumentedDTO +from my_ast.persistence.file_content_dto import FileContentDTO + + +@unique +class ParseStatus(Enum): + PSNONE = 0 + PSPARTIALLYPARSED = 1 + PSFULLYPARSED = 2 + + +class FileDTO(DocumentedDTO): + def __init__(self): + super().__init__("") + self.type: str = 'Python' # dir, unknown, binary? + self.path: str = "" + self.file_name: str = "" + self.timestamp = "" + self.content: FileContentDTO = FileContentDTO("") + self.parent: str = "" + self.parse_status: int = 0 # PSNONE diff --git a/plugins/python/parser/src/my_ast/persistence/function_dto.py b/plugins/python/parser/src/my_ast/persistence/function_dto.py new file mode 100644 index 000000000..af7a450f9 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/function_dto.py @@ -0,0 +1,12 @@ +from typing import Set, List + +from my_ast.common.file_position import FilePosition +from my_ast.persistence.base_dto import DeclarationDTO, UsageDTO +from my_ast.persistence.documented_dto import DocumentedDTO + + +class FunctionDeclarationDTO(DeclarationDTO, DocumentedDTO): + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO], documentation: str): + DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) + DocumentedDTO.__init__(self, documentation) diff --git a/plugins/python/parser/src/my_ast/persistence/import_dto.py b/plugins/python/parser/src/my_ast/persistence/import_dto.py new file mode 100644 index 000000000..c514fa4d4 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/import_dto.py @@ -0,0 +1,17 @@ +from pathlib import PurePath +from typing import Set, Dict + + +class ImportDTO: + def __init__(self, importer: PurePath): + self.importer: str = str(importer) + self.imported_modules: Set[str] = set() + self.imported_symbols: Dict[str, Set[str]] = {} + + def add_module_import(self, imported: PurePath): + self.imported_modules.add(str(imported)) + + def add_symbol_import(self, imported: PurePath, symbol_qualified_name: str): + if str(imported) not in self.imported_symbols: + self.imported_symbols[str(imported)] = set() + self.imported_symbols[str(imported)].add(symbol_qualified_name) diff --git a/plugins/python/parser/src/my_ast/persistence/persistence.py b/plugins/python/parser/src/my_ast/persistence/persistence.py new file mode 100644 index 000000000..5248d7937 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/persistence.py @@ -0,0 +1,48 @@ +from my_ast.persistence.file_dto import FileDTO +from my_ast.persistence.variable_dto import VariableDeclarationDTO +from my_ast.persistence.function_dto import FunctionDeclarationDTO +from my_ast.persistence.class_dto import ClassDeclarationDTO +from my_ast.persistence.import_dto import ImportDTO + + +class ModelPersistence: + def __init__(self, c_persistence): + self.c_persistence = c_persistence + self.check_c_persistence() + + def check_c_persistence(self): + if self.c_persistence is None: + return + assert hasattr(self.c_persistence, 'persist_file') + assert hasattr(self.c_persistence, 'persist_variable') + assert hasattr(self.c_persistence, 'persist_function') + assert hasattr(self.c_persistence, 'persist_class') + assert hasattr(self.c_persistence, 'persist_module_import') + + def persist_file(self, file: FileDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.perist_file(file) + + def persist_variable(self, declaration: VariableDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.persist_variable(declaration) + + def persist_function(self, declaration: FunctionDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.persist_function(declaration) + + def persist_class(self, declaration: ClassDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.persist_class(declaration) + + def persist_import(self, imports: ImportDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.persist_module_import(imports) + + +model_persistence = ModelPersistence(None) + + +def init_persistence(c_persistence): + global model_persistence + model_persistence = ModelPersistence(c_persistence) diff --git a/plugins/python/parser/src/my_ast/persistence/variable_dto.py b/plugins/python/parser/src/my_ast/persistence/variable_dto.py new file mode 100644 index 000000000..0e3b7bca5 --- /dev/null +++ b/plugins/python/parser/src/my_ast/persistence/variable_dto.py @@ -0,0 +1,5 @@ +from my_ast.persistence.base_dto import DeclarationDTO + + +class VariableDeclarationDTO(DeclarationDTO): + pass diff --git a/plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py b/plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py new file mode 100644 index 000000000..85b917a3d --- /dev/null +++ b/plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py @@ -0,0 +1,49 @@ +import ast +from typing import Dict, Tuple, Optional, Set, Union, List + +from my_ast.common.hashable_list import OrderedHashableList +from my_ast.function_data import FunctionDeclaration +from my_ast.type_data import DeclarationType + + +# TODO: redefined functions? +class PlaceholderFunctionDeclarationCache: + def __init__(self): + self.cache: \ + Dict[FunctionDeclaration, + Tuple[Union[ast.FunctionDef, ast.AsyncFunctionDef], + Dict[OrderedHashableList[DeclarationType], Set[DeclarationType]]]] = {} + + def add_function_declaration(self, declaration: FunctionDeclaration, + function_def: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: + self.cache[declaration] = (function_def, {}) + + def get_function_def(self, declaration: FunctionDeclaration) \ + -> Optional[Union[ast.FunctionDef, ast.AsyncFunctionDef]]: + if declaration in self.cache: + return self.cache[declaration][0] + return None + + def get_function_return_type(self, declaration: FunctionDeclaration, + param_types: OrderedHashableList[DeclarationType]) -> Optional[Set[DeclarationType]]: + if declaration in self.cache and param_types in self.cache[declaration][1]: + return self.cache[declaration][1][param_types] + return None + + def get_functions_return_type(self, declarations: Set[FunctionDeclaration], + param_types: OrderedHashableList[DeclarationType]) \ + -> Set[DeclarationType]: + types = set() + for declaration in declarations: + t = self.get_function_return_type(declaration, param_types) + if t is not None: + types.update(t) + return types + + def add_function_return_type(self, declaration: FunctionDeclaration, + param_types: OrderedHashableList[DeclarationType], return_type: Set[DeclarationType]) \ + -> None: + if declaration in self.cache and param_types not in self.cache[declaration][1]: + self.cache[declaration][1][param_types] = return_type + elif declaration not in self.cache: + assert False, "Key not found" diff --git a/plugins/python/parser/src/my_ast/preprocessed_data.py b/plugins/python/parser/src/my_ast/preprocessed_data.py new file mode 100644 index 000000000..f12ddd2b6 --- /dev/null +++ b/plugins/python/parser/src/my_ast/preprocessed_data.py @@ -0,0 +1,8 @@ +from my_ast.base_data import Declaration +from my_ast.common.file_position import FilePosition + + +class PreprocessedDeclaration(Declaration): + def __init__(self, name: str): + file_position = FilePosition.get_empty_file_position() + Declaration.__init__(self, name, "", file_position) # TODO: need qualified name? diff --git a/plugins/python/parser/src/my_ast/preprocessed_file.py b/plugins/python/parser/src/my_ast/preprocessed_file.py new file mode 100644 index 000000000..2219e7dbe --- /dev/null +++ b/plugins/python/parser/src/my_ast/preprocessed_file.py @@ -0,0 +1,59 @@ +import ast +from typing import Set, Any + +from my_ast.class_preprocessor import PreprocessedClassCollector +from my_ast.import_preprocessor import ImportTable +from my_ast.preprocessed_function import PreprocessedFunction +from my_ast.preprocessed_variable import PreprocessedVariable + + +class PreprocessedFile(ast.NodeVisitor): + def __init__(self): + self.class_collector = PreprocessedClassCollector() + self.preprocessed_functions: Set[PreprocessedFunction] = set() + self.preprocessed_variables: Set[PreprocessedVariable] = set() + self.import_table = ImportTable() + self.depth = 0 + self.documentation: str = "" + + def visit_Module(self, node: ast.Module) -> Any: + documentation = ast.get_docstring(node) + if documentation is not None: + self.documentation = documentation + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> Any: + is_local = False + if self.depth > 0: + is_local = True + self.import_table.append_import(node, is_local) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: + is_local = False + if self.depth > 0: + is_local = True + self.import_table.append_import_from(node, is_local) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> Any: + self.class_collector.append_class(node) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + self.class_collector.class_processed() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + if self.depth == 0: + self.preprocessed_functions.add(PreprocessedFunction(node.name)) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_Assign(self, node: ast.Assign) -> Any: + if self.depth == 0: + for target in node.targets: + if isinstance(target, ast.Name): + self.preprocessed_variables.add(PreprocessedVariable(target.id)) + else: + print("...") diff --git a/plugins/python/parser/src/my_ast/preprocessed_function.py b/plugins/python/parser/src/my_ast/preprocessed_function.py new file mode 100644 index 000000000..c0531a8a3 --- /dev/null +++ b/plugins/python/parser/src/my_ast/preprocessed_function.py @@ -0,0 +1,19 @@ +from typing import List + +from my_ast.base_data import Declaration +from my_ast.preprocessed_data import PreprocessedDeclaration + + +class PreprocessedFunction(PreprocessedDeclaration): + def __init__(self, name): + super().__init__(name) + self.type_usages: List[Declaration] = [] + + def __eq__(self, other): + return isinstance(other, PreprocessedFunction) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/my_ast/preprocessed_variable.py b/plugins/python/parser/src/my_ast/preprocessed_variable.py new file mode 100644 index 000000000..a4ebd1ee4 --- /dev/null +++ b/plugins/python/parser/src/my_ast/preprocessed_variable.py @@ -0,0 +1,19 @@ +from typing import List + +from my_ast.base_data import Declaration +from my_ast.preprocessed_data import PreprocessedDeclaration + + +class PreprocessedVariable(PreprocessedDeclaration): + def __init__(self, name): + super().__init__(name) + self.type_usages: List[Declaration] = [] + + def __eq__(self, other): + return isinstance(other, PreprocessedVariable) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/my_ast/python_parser.py b/plugins/python/parser/src/my_ast/python_parser.py new file mode 100644 index 000000000..0c3db8f6c --- /dev/null +++ b/plugins/python/parser/src/my_ast/python_parser.py @@ -0,0 +1,30 @@ +import os +from pathlib import PurePath + +from my_ast.parse_exception import ParseException +from my_ast.parser import Parser +from my_ast.persistence.persistence import init_persistence + + +def parse(source_root: str, persistence): + print(source_root) + + print(persistence.f()) + + init_persistence(persistence) + + def directory_exception(path: PurePath) -> bool: + directory = os.path.basename(os.path.normpath(str(path))) + return directory.startswith('.') or directory == 'venv' + + def file_exception(path: PurePath) -> bool: + return False + + exception = ParseException(directory_exception, file_exception) + p = Parser([source_root], exception) + p.parse() + pass + + +if __name__ == '__main__': + parse("dummy", None) # error! diff --git a/plugins/python/parser/src/my_ast/scope.py b/plugins/python/parser/src/my_ast/scope.py new file mode 100644 index 000000000..4014cb6ae --- /dev/null +++ b/plugins/python/parser/src/my_ast/scope.py @@ -0,0 +1,177 @@ +from typing import List, Optional +from abc import abstractmethod, ABC +from uuid import uuid4 + +from my_ast.base_data import Declaration, Usage +from my_ast.class_data import ClassDeclaration +from my_ast.import_preprocessor import ImportTable +from my_ast.logger import logger +from my_ast.function_data import FunctionDeclaration +from my_ast.type_data import InitVariablePlaceholderType +from my_ast.variable_data import VariableDeclaration + + +class Scope(ABC): + def __init__(self, name: str): + self.name: str = name + self.variable_declarations: List[VariableDeclaration] = [] + self.function_declarations: List[FunctionDeclaration] = [] + self.class_declarations: List[ClassDeclaration] = [] + # TODO: should use only self.declarations, and filter the others? (slower, but less memory) + self.declarations: List[Declaration] = [] + + def append_variable(self, variable_declaration: VariableDeclaration) -> None: + self.variable_declarations.append(variable_declaration) + self.declarations.append(variable_declaration) + logger.debug('Var: ' + variable_declaration.name + ' (' + str(variable_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ' + variable_declaration.get_type_repr()) + + def append_function(self, function_declaration: FunctionDeclaration) -> None: + self.function_declarations.append(function_declaration) + self.declarations.append(function_declaration) + logger.debug('Func: ' + function_declaration.name + ' (' + str(function_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ' + function_declaration.get_type_repr()) + + def append_class(self, class_declaration: ClassDeclaration) -> None: + self.class_declarations.append(class_declaration) + self.declarations.append(class_declaration) + logger.debug('Class: ' + class_declaration.name + ' (' + str(class_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ') + + def get_variable(self, usage: Usage) -> Optional[VariableDeclaration]: + return self.get_variable_by_name(usage.name) + + def get_variable_by_name(self, name: str) -> Optional[VariableDeclaration]: + variables = [x for x in self.variable_declarations if x.name == name] + # TODO: fix this assert + # assert len(variables) <= 1 + if len(variables) > 0: + return variables[0] + else: + return None + + def get_function(self, usage: Usage) -> Optional[FunctionDeclaration]: + return self.get_function_by_name(usage.name) + + def get_function_by_name(self, name: str) -> Optional[FunctionDeclaration]: + functions = [x for x in self.function_declarations if x.name == name] + if len(functions) > 0: + return functions[-1] + else: + return None + + def get_class_by_name(self, name: str) -> Optional[ClassDeclaration]: + classes = [x for x in self.class_declarations if x.name == name] + if len(classes) > 0: + return classes[-1] + else: + return None + + @abstractmethod + def get_type(self) -> str: + pass + + +class LifetimeScope(Scope, ABC): + """Symbols declared inside this scope are deleted after exiting from it.""" + + def __init__(self, name: str): + super().__init__(name) + + +class ImportScope(ABC): + """Imported modules are available during this scope.""" + + def __init__(self): + self.import_table: ImportTable = ImportTable() + + +class GlobalScope(LifetimeScope, ImportScope): + """The first scope, which lives on with its module.""" + + def __init__(self): + LifetimeScope.__init__(self, '') + ImportScope.__init__(self) + + def get_type(self) -> str: + return "global" + + +class FunctionScope(LifetimeScope, ImportScope): + """Scope of the functions and methods.""" + + def __init__(self, name: str): + LifetimeScope.__init__(self, name) + ImportScope.__init__(self) + + def get_type(self) -> str: + return "function" + + +class ClassScope(LifetimeScope, ImportScope): + """Scope of the classes.""" + + def __init__(self, name: str): + LifetimeScope.__init__(self, name) + ImportScope.__init__(self) + self.init_placeholders: List[InitVariablePlaceholderType] = [] + + def get_type(self) -> str: + return "class" + + +class PartialLifetimeScope(LifetimeScope, ABC): + """Not all of the symbols are deleted after existing from it.""" + pass + + +class ExceptionScope(PartialLifetimeScope): + """The exception variable is deleted after the 'except' scope, + but all the other symbols declared in it, are alive after it.""" + + def __init__(self): + super().__init__('exception') + + def get_type(self) -> str: + return 'exception' + + +class LambdaScope(LifetimeScope): + """Scope of lambdas.""" + + def __init__(self): + super().__init__('lambda_' + uuid4().hex) + + def get_type(self) -> str: + return "lambda" + + +class GeneratorScope(LifetimeScope): + """Scope of generators.""" + + def __init__(self): + super().__init__('generator_' + uuid4().hex) + + def get_type(self) -> str: + return "generator" + + +class ConditionalScope(Scope): + """Scope of if.""" + + def __init__(self): + super().__init__('if') + + def get_type(self) -> str: + return "conditional" + + +# TODO: need WhileScope and ForScope? +class LoopScope(Scope): + """Scope of for and while.""" + + def __init__(self): + super().__init__('loop') + + def get_type(self) -> str: + return "loop" diff --git a/plugins/python/parser/src/my_ast/scope_manager.py b/plugins/python/parser/src/my_ast/scope_manager.py new file mode 100644 index 000000000..9f4b3f342 --- /dev/null +++ b/plugins/python/parser/src/my_ast/scope_manager.py @@ -0,0 +1,383 @@ +import ast +from pathlib import PurePath, Path +from typing import List, Type, Iterator, Callable, Optional, Union + +from my_ast.base_data import Declaration, ImportedDeclaration +from my_ast.class_data import ClassDeclaration +from my_ast.function_data import FunctionDeclaration +from my_ast.import_finder import ImportFinder +from my_ast.import_preprocessor import ImportTable +from my_ast.member_access_collector import MemberAccessCollectorIterator, MemberAccessCollector +from my_ast.persistence.import_dto import ImportDTO +from my_ast.persistence.persistence import model_persistence +from my_ast.placeholder_function_declaration_cache import PlaceholderFunctionDeclarationCache +from my_ast.scope import Scope, ClassScope, GlobalScope, FunctionScope, LifetimeScope, PartialLifetimeScope, \ + ConditionalScope, LoopScope, ImportScope +from my_ast.type_data import PlaceholderType +from my_ast.variable_data import VariableDeclaration, ModuleVariableDeclaration + + +class ScopeManager: + def __init__(self, current_file: PurePath, import_finder: ImportFinder, scopes: Optional[List[Scope]] = None): + self.current_file = current_file + self.import_finder = import_finder + if scopes is None: + self.scopes: List[Scope] = [GlobalScope()] + else: + self.scopes = scopes + self.placeholder_function_declaration_cache = PlaceholderFunctionDeclarationCache() + + def is_current_scope_instance(self, t: Type) -> bool: + return isinstance(self.scopes[-1], t) + + # TODO: local functions + def is_current_scope_method(self) -> bool: + for i in range(len(self.scopes) - 1, -1, -1): + if isinstance(self.scopes[i], LifetimeScope) and not isinstance(self.scopes[i], PartialLifetimeScope): + return isinstance(self.scopes[i], FunctionScope) and isinstance(self.scopes[i - 1], ClassScope) + return False + + def is_current_scope_init(self) -> bool: + return self.is_current_scope_method() and self.get_current_lifetime_scope().name == '__init__' + + def get_current_lifetime_scope(self) -> LifetimeScope: + for scope in self.reverse(): + if isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): + return scope + assert False # there should be always a GlobalScope, which is LifetimeScope + + def reverse(self) -> Iterator[Scope]: + for scope in reversed(self.scopes): + yield scope + + def iterate(self) -> Iterator[Scope]: + for scope in self.scopes: + yield scope + + def with_scope(self, scope: Scope, action: Callable[[], None]) -> None: + self.scopes.append(scope) + action() + self.persist_current_scope() + del self.scopes[-1] + + def get_size(self) -> int: + return len(self.scopes) + + def get_current_scope(self) -> Scope: + return self.scopes[-1] + + def get_global_scope(self) -> GlobalScope: + assert len(self.scopes) > 0 + global_scope = self.scopes[0] + if not isinstance(global_scope, GlobalScope): + assert False + return global_scope + + def append_variable_to_current_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_lifetime_scope().append_variable(new_variable) + + def append_exception_to_current_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_partial_lifetime_scope().append_variable(new_variable) + + def append_function_to_current_scope(self, new_function: FunctionDeclaration) -> None: + self.get_current_lifetime_scope().append_function(new_function) + + def append_class_to_current_scope(self, new_class: ClassDeclaration) -> None: + self.get_current_lifetime_scope().append_class(new_class) + + def append_variable_to_current_class_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_class_scope().append_variable(new_variable) + + def append_function_to_current_class_scope(self, new_function: FunctionDeclaration) -> None: + self.get_current_class_scope().append_function(new_function) + + def append_class_to_current_class_scope(self, new_class: ClassDeclaration) -> None: + self.get_current_class_scope().append_class(new_class) + + def append_module_variable_to_current_scope(self, module_variable: ModuleVariableDeclaration) -> None: + def is_same_declaration(var: VariableDeclaration) -> bool: + return isinstance(var, ModuleVariableDeclaration) and \ + var.imported_module_location == module_variable.imported_module_location + scope = self.get_current_lifetime_scope() + if len([v for v in scope.variable_declarations if is_same_declaration(v)]) == 0: + scope.append_variable(module_variable) + + def get_current_partial_lifetime_scope(self) -> Optional[LifetimeScope]: + for scope in self.reverse(): + if isinstance(scope, PartialLifetimeScope): + return scope + return None + + def get_current_class_scope(self) -> Optional[ClassScope]: + for scope in self.reverse(): + if isinstance(scope, ClassScope): + return scope + return None + + def get_current_scope_name(self) -> str: + return self.scopes[-1].name + + def is_inside_conditional_scope(self) -> bool: + for scope in self.reverse(): + if isinstance(scope, (ConditionalScope, LoopScope)): + return True + elif isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): + return False + assert False # GlobalScope is LifetimeScope + + def is_declaration_in_current_scope(self, name: str) -> bool: + return self.is_declaration_in_scope(name, self.scopes[-1]) + + def is_declaration_in_current_class(self, name: str) -> bool: + class_scope = self.get_current_class_scope() + if class_scope is not None: + return self.is_declaration_in_scope(name, class_scope) + return False + + def get_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: + for scope in self.reverse(): + declaration = self.get_declaration_from_scope(name, scope) + if declaration is not None: + from my_ast.variable_data import TypeVariable + if isinstance(declaration, TypeVariable): + print("NEED FIX: TypeVariable") + return declaration + current_class_scope = self.get_current_class_scope() + if current_class_scope is not None: + for init_placeholder in current_class_scope.init_placeholders: + if name == init_placeholder.name: + return init_placeholder + return PlaceholderType(name) + + def get_function_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: + for scope in self.reverse(): + declaration = self.get_function_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_declaration_from_member_access(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: + iter(iterator) + members = [next(iterator)] + # TODO: MethodVariable? + if isinstance(members[0], MemberAccessCollector.MethodData): + declaration = self.get_function_declaration(members[0].name) + else: + declaration = self.get_declaration(members[0].name) + if not isinstance(declaration, PlaceholderType): + return declaration + global_scope = self.get_global_scope() + assert isinstance(global_scope, GlobalScope) + try: + module_variable = self.get_imported_module(iterator) + if module_variable is not None: + declaration = module_variable + else: + declaration_from_other_module = self.get_declaration_from_other_module(iterator) + if declaration_from_other_module is not None: + declaration = declaration_from_other_module + except StopIteration: + assert False + return declaration + + # import a.b.c -> c.f() vs a.b.c.f() + def get_declaration_from_other_module(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + # TODO: is this alias check enough? + if module.module.alias is None and len(module.path) >= len(iterator.mac.call_list): + continue + is_same = True + current = next(iterator) + name = current.name + iter(iterator) + if module.module.alias is not None: + is_same = module.module.alias == name + else: + for i in range(0, len(module.path)): + name = next(iterator).name + if module.path[i] != name: + is_same = False + break + if is_same: + name = next(iterator).name + if module.is_module_import(): + other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) + if other_scope_manager is not None: + if isinstance(current, MemberAccessCollector.MemberData): + return other_scope_manager.get_function_declaration(name) + else: + return other_scope_manager.get_declaration(name) + for imp in module.imported: + if (imp.alias is not None and imp.alias == name) \ + or imp.name == name or imp.is_all_imported(): + other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) + if other_scope_manager is not None: + if isinstance(current, MemberAccessCollector.MemberData): + return other_scope_manager.get_function_declaration(name) + else: + return other_scope_manager.get_declaration(name) + return None + + def get_imported_module(self, iterator: MemberAccessCollectorIterator) -> Optional[ModuleVariableDeclaration]: + imported_module = None + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + if len(module.path) != len(iterator.mac.call_list): + continue + for i in range(0, len(module.path)): + next(iterator) + module_path = [m.name for m in iterator.mac.call_list] + module_path.reverse() + if module.path == module_path: + imported_module = module + break + if imported_module is not None and isinstance(scope, Scope): + for var in reversed(scope.variable_declarations): + if isinstance(var, ModuleVariableDeclaration) and \ + var.imported_module_location == imported_module.location: + return var + return None + + def get_nonlocal_variable(self, name: str) -> Optional[VariableDeclaration]: + assert len(self.scopes) > 2 + for scope in self.scopes[-2:0:-1]: + var = scope.get_variable_by_name(name) + if var is not None: + return var + return None + + def get_global_variable(self, name: str) -> Optional[VariableDeclaration]: + return self.scopes[0].get_variable_by_name(name) + + @staticmethod + def get_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: + for declaration in reversed(scope.declarations): + if declaration.name == name: + return declaration + return None + + @staticmethod + def get_function_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: + for func in reversed(scope.function_declarations): + if func.name == name: + return func + return None + + @staticmethod + def is_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return True + return False + + def get_current_class_declaration(self) -> Optional[ClassDeclaration]: + current_class_scope_found = False + for scope in self.reverse(): + if current_class_scope_found: + return scope.class_declarations[-1] + current_class_scope_found = isinstance(scope, ClassScope) + return None + + def set_global_import_table(self, import_table: ImportTable): + global_scope = self.scopes[0] + if isinstance(global_scope, GlobalScope): + global_scope.import_table = import_table + + def append_import(self, node: ast.Import): + for scope in self.reverse(): + if isinstance(scope, ImportScope): + scope.import_table.append_import(node, isinstance(scope, GlobalScope)) + return + assert False + + def append_import_from(self, node: ast.ImportFrom): + for scope in self.reverse(): + if isinstance(scope, ImportScope): + scope.import_table.append_import_from(node, isinstance(scope, GlobalScope)) + return + assert False + + def handle_import_scope(self): + if isinstance(self.scopes[-1], ImportScope): + for declaration in self.scopes[-1].declarations: + if isinstance(declaration, ImportedDeclaration): + declaration.imported_declaration.usages.extend(declaration.usages) + + def persist_current_scope(self): + # for declaration in self.scopes[-1].declarations: + # print(self.get_qualified_name_from_current_scope(declaration)) + + self.handle_import_scope() + + import_dto = ImportDTO(self.current_file) + + for var in self.scopes[-1].variable_declarations: + if isinstance(var, ImportedDeclaration): + import_dto.add_symbol_import(var.imported_declaration.file_position.file, + var.imported_declaration.qualified_name) + elif isinstance(var, ModuleVariableDeclaration): + import_dto.add_module_import(var.imported_module_location) + else: + model_persistence.persist_variable(var.create_dto()) + + for func in self.scopes[-1].function_declarations: + if isinstance(func, ImportedDeclaration): + import_dto.add_symbol_import(func.imported_declaration.file_position.file, + func.imported_declaration.qualified_name) + else: + model_persistence.persist_function(func.create_dto()) + + for cl in self.scopes[-1].class_declarations: + if isinstance(cl, ImportedDeclaration): + import_dto.add_symbol_import(cl.imported_declaration.file_position.file, + cl.imported_declaration.qualified_name) + else: + model_persistence.persist_class(cl.create_dto()) + + if len(import_dto.imported_modules) > 0 or len(import_dto.imported_symbols) > 0: + model_persistence.persist_import(import_dto) + + def get_qualified_name_from_current_scope(self, declaration_name: str) -> str: + qualified_name_parts = [declaration_name] + for scope in reversed(self.scopes): + if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope + qualified_name_parts.append(scope.name) + file_name = self.current_file.name + assert file_name[-3:] == '.py' + qualified_name_parts.append(file_name[:-3]) + directory = self.current_file.parent + while True: + init_file_location = Path(directory).joinpath("__init__.py") + if init_file_location.exists(): + qualified_name_parts.append(directory.name) + directory = directory.parent + else: + break + + return '.'.join(reversed(qualified_name_parts)) + + # @staticmethod + # def is_variable_declaration_in_scope(name: str, scope: Scope) -> bool: + # for var in scope.declarations: + # if var.name == name: + # return isinstance(var, VariableDeclaration) + # return False + # + # @staticmethod + # def is_function_declaration_in_scope(name: str, scope: Scope) -> bool: + # for var in scope.declarations: + # if var.name == name: + # return isinstance(var, FunctionDeclaration) + # return False + # + # @staticmethod + # def is_class_declaration_in_scope(name: str, scope: Scope) -> bool: + # for var in scope.declarations: + # if var.name == name: + # return isinstance(var, ClassDeclaration) + # return False diff --git a/plugins/python/parser/src/my_ast/symbol_collector.py b/plugins/python/parser/src/my_ast/symbol_collector.py new file mode 100644 index 000000000..a6784222e --- /dev/null +++ b/plugins/python/parser/src/my_ast/symbol_collector.py @@ -0,0 +1,881 @@ +import ast +from pathlib import PurePath +from typing import List, Optional, Any, Union, Set, TypeVar + +from my_ast.built_in_functions import get_built_in_function +from my_ast.common.file_position import FilePosition +from my_ast.common.utils import create_range_from_ast_node +from my_ast.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from my_ast.symbol_collector_interface import SymbolCollectorBase +from my_ast.base_data import Declaration, Usage +from my_ast.class_data import ClassDeclaration, ImportedClassDeclaration +from my_ast.class_init_declaration import ClassInitDeclaration +from my_ast.class_preprocessor import PreprocessedClass +from my_ast.common.parser_tree import ParserTree +from my_ast.common.position import Range +from my_ast.import_finder import ImportFinder +from my_ast.import_preprocessor import ImportTable +from my_ast.logger import logger +from my_ast.member_access_collector import MemberAccessCollector +from my_ast.preprocessed_file import PreprocessedFile +from my_ast.preprocessed_function import PreprocessedFunction +from my_ast.preprocessed_variable import PreprocessedVariable +from my_ast.scope import Scope, ClassScope, FunctionScope, GeneratorScope, LambdaScope, \ + ExceptionScope, ConditionalScope, LoopScope, GlobalScope +from my_ast.function_data import FunctionDeclaration, ImportedFunctionDeclaration, \ + StaticFunctionDeclaration +from my_ast.scope_manager import ScopeManager +from my_ast.symbol_finder import SymbolFinder +from my_ast.type_data import PlaceholderType, InitVariablePlaceholderType +from my_ast.type_deduction import TypeDeduction, VariablePlaceholderType +from my_ast.variable_data import VariableDeclaration, ImportedVariableDeclaration, \ + ModuleVariableDeclaration, StaticVariableDeclaration, NonlocalVariableDeclaration, GlobalVariableDeclaration, \ + FunctionVariable, TypeVariable + + +class SymbolCollector(ast.NodeVisitor, SymbolFinder, SymbolCollectorBase): + def __init__(self, tree: ParserTree, current_file: PurePath, preprocessed_file: PreprocessedFile, + import_finder: ImportFinder, function_symbol_collector_factory: FunctionSymbolCollectorFactory): + SymbolCollectorBase.__init__(self, preprocessed_file, import_finder) + self.tree = tree + self.current_file = current_file + self.scope_manager: ScopeManager = ScopeManager(current_file, import_finder) + self.scope_manager.set_global_import_table(preprocessed_file.import_table) + self.current_function_declaration: List[FunctionDeclaration] = [] + self.function_symbol_collector_factory = function_symbol_collector_factory + self.type_deduction = TypeDeduction(self, self.scope_manager, self.function_symbol_collector_factory) + + def collect_symbols(self): + self.visit(self.tree.root.node) + self.post_process() + + def visit_Import(self, node: ast.Import) -> Any: + modules = ImportTable.convert_ast_import_to_import(node) + for module in modules: + if not module.is_module_import(): + continue + # scope.imports.append((module, self.import_finder.get_global_scope_by_location(module.location))) + self.scope_manager.append_module_variable_to_current_scope( + ModuleVariableDeclaration(module.path[-1], module.location, + FilePosition(self.current_file, module.range), + self.import_finder.get_global_scope_by_location(module.location))) + if len(self.scope_manager.scopes) > 1: + self.scope_manager.append_import(node) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: + modules = ImportTable.convert_ast_import_from_to_import(node) + for module in modules: + self.scope_manager.append_module_variable_to_current_scope( + ModuleVariableDeclaration(module.path[-1], module.location, + FilePosition(self.current_file, module.range), + self.import_finder.get_global_scope_by_location(module.location))) + if not module.is_module_import(): + scope_manager: ScopeManager = self.import_finder.get_scope_manager_by_location(module.location) + if scope_manager is None: + # import from library + continue + # in case of circular import this case is not valid (runtime error) -> must find declaration + file_position = FilePosition(self.current_file, Range.get_empty_range()) + for imported in module.imported: + if imported.is_all_imported(): + for declaration in scope_manager.get_global_scope().declarations: + if isinstance(declaration, VariableDeclaration): + var = ImportedVariableDeclaration( + declaration.name, file_position, declaration) + self.scope_manager.append_variable_to_current_scope(var) + self.imported_declaration_scope_map[var] = scope_manager + elif isinstance(declaration, FunctionDeclaration): + func = ImportedFunctionDeclaration( + declaration.name, file_position, declaration) + self.scope_manager.append_function_to_current_scope(func) + self.imported_declaration_scope_map[func] = scope_manager + elif isinstance(declaration, ClassDeclaration): + c = ImportedClassDeclaration( + declaration.name, file_position, declaration) + self.scope_manager.append_class_to_current_scope(c) + self.imported_declaration_scope_map[c] = scope_manager + else: + assert False + else: + # TODO: redefinition? + declaration = scope_manager.get_function_declaration(imported.name) + if isinstance(declaration, PlaceholderType): + declaration = scope_manager.get_declaration(imported.name) + if isinstance(declaration, PlaceholderType): + continue + elif isinstance(declaration, VariableDeclaration): + var = ImportedVariableDeclaration( + imported.name if imported.alias is None else imported.alias, + file_position, declaration) + self.scope_manager.append_variable_to_current_scope(var) + self.imported_declaration_scope_map[var] = scope_manager + elif isinstance(declaration, FunctionDeclaration): + func = ImportedFunctionDeclaration( + imported.name if imported.alias is None else imported.alias, + file_position, declaration) + self.scope_manager.append_function_to_current_scope(func) + self.imported_declaration_scope_map[func] = scope_manager + elif isinstance(declaration, ClassDeclaration): + c = ImportedClassDeclaration( + imported.name if imported.alias is None else imported.alias, + file_position, declaration) + self.scope_manager.append_class_to_current_scope(c) + self.imported_declaration_scope_map[c] = scope_manager + else: + assert False + if len(self.scope_manager.scopes) > 1: + self.scope_manager.append_import_from(node) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> Any: + base_classes = self.collect_base_classes(node) + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name) + new_class = ClassDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), + base_classes, self.get_documentation(node)) + self.class_def_process_derived_members(new_class, base_classes) + # TODO: handle preprocessed classes + self.scope_manager.get_current_scope().append_class(new_class) + # TODO: check if has at least 1 parameter? (self) + # TODO: range could be the __init__ + self.scope_manager.append_function_to_current_scope( + ClassInitDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), [], new_class)) + self.scope_manager.append_variable_to_current_scope( + TypeVariable(node.name, qualified_name, FilePosition(self.current_file, r), new_class)) + + def action(): + self.generic_visit(node) + scope = self.scope_manager.get_current_scope() + assert isinstance(scope, ClassScope) + for var in scope.variable_declarations: + if isinstance(var, StaticVariableDeclaration) and var not in new_class.static_attributes: + new_class.static_attributes.append(var) + elif isinstance(var, VariableDeclaration) and var not in new_class.attributes: + new_class.attributes.append(var) + for func in scope.function_declarations: + if isinstance(func, StaticFunctionDeclaration): + new_class.static_methods.append(func) + elif isinstance(func, FunctionDeclaration): + new_class.methods.append(func) + else: + assert False + for c in scope.class_declarations: + new_class.classes.append(c) + + self.scope_manager.with_scope(ClassScope(node.name), action) + self.class_def_post_process(new_class) + + def collect_base_classes(self, node: ast.ClassDef) -> List[ClassDeclaration]: + base_classes = [] + for base_class in node.bases: + # declaration = None + base_class_types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(base_class, self.preprocessed_file)) + # if isinstance(base_class, ast.Name): + # declaration = self.scope_manager.get_declaration(base_class.id) + # elif isinstance(base_class, ast.Subscript): + # pass + # # declaration = self.scope_manager.get_declaration(base_class.value.id) + # elif isinstance(base_class, ast.Attribute): + # pass + # else: + # assert False + assert len(base_class_types) <= 1 + for base_class_type in base_class_types: + if isinstance(base_class_type, ClassDeclaration): + base_classes.append(base_class_type) + elif isinstance(base_class_type, TypeVariable): + base_classes.append(base_class_type.reference) + else: + print('') + return base_classes + + def class_def_process_derived_members(self, cl: ClassDeclaration, base_classes: List[ClassDeclaration]): + declaration_type = TypeVar('declaration_type', bound=Declaration) + + def collect_derived_members(members: List[List[declaration_type]]) -> List[declaration_type]: + declarations = [] + for base_class_members in members: + for member in base_class_members: + if all([m.name != member.name for m in declarations]): + declarations.append(member) + return declarations + + cl.attributes.extend(collect_derived_members([d.attributes for d in base_classes])) + cl.static_attributes.extend(collect_derived_members([d.static_attributes for d in base_classes])) + cl.methods.extend(collect_derived_members([d.methods for d in base_classes])) + cl.static_methods.extend(collect_derived_members([d.static_methods for d in base_classes])) + cl.classes.extend(collect_derived_members([d.classes for d in base_classes])) + + def class_def_post_process(self, cl: ClassDeclaration): + def remove_hidden_declarations(declarations: List[Declaration]): + for declaration in reversed(declarations): + hidden_declarations = [d for d in declarations if d.name == declaration.name and d is not declaration] + for hidden_declaration in hidden_declarations: + declarations.remove(hidden_declaration) + + remove_hidden_declarations(cl.attributes) + remove_hidden_declarations(cl.static_attributes) + remove_hidden_declarations(cl.methods) # TODO: singledispatchmethod + remove_hidden_declarations(cl.static_methods) + remove_hidden_declarations(cl.classes) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + self.visit_common_function_def(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: + self.visit_common_function_def(node) + + def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name) + if self.is_static_method(node): + new_function = StaticFunctionDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), + [], self.get_documentation(node)) + else: + new_function = FunctionDeclaration(node.name, qualified_name, + FilePosition(self.current_file, r), [], self.get_documentation(node)) + self.scope_manager.append_function_to_current_scope(new_function) + new_func_var = FunctionVariable(node.name, qualified_name, FilePosition(self.current_file, r), new_function) + self.scope_manager.append_variable_to_current_scope(new_func_var) + self.current_function_declaration.append(new_function) + + # if node.name == '__init__' and isinstance(self.scopes[-1], ClassScope): # ctor ~ __init__ + + def action(): + # self.collect_parameters(node) + self.process_function_def(node) + if any(isinstance(x, PlaceholderType) for x in self.current_function_declaration[-1].type): + self.scope_manager.placeholder_function_declaration_cache. \ + add_function_declaration(new_function, node) + + self.scope_manager.with_scope(FunctionScope(node.name), action) + del self.current_function_declaration[-1] + + def get_documentation(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef]) -> str: + documentation = ast.get_docstring(node) + if documentation is None: + return "" + else: + return documentation + + def collect_parameters(self, node: ast.FunctionDef) -> None: + for param in node.args.args: + if hasattr(param, 'arg'): # ast.Name + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(param.arg) + new_variable = VariableDeclaration(param.arg, qualified_name, FilePosition(self.current_file, r)) + # TODO: first parameter could be anything, not just self + static method check! + if param.arg == 'self' and self.scope_manager.is_current_scope_method() and node.args.args[0] is param: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + elif node.name == '__init__': + init_var_placeholder_type = InitVariablePlaceholderType(param.arg) + new_variable.type.add(init_var_placeholder_type) + self.scope_manager.get_current_class_scope().init_placeholders.append(init_var_placeholder_type) + else: + new_variable.type.add(VariablePlaceholderType(param.arg)) + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + assert False, "Parameter with unknown variable name" + + def process_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: + def get_first_param() -> Optional[ast.arg]: + if len(node.args.posonlyargs) > 0: + return node.args.posonlyargs[0] + if len(node.args.args) > 0: + return node.args.args[0] + if node.args.vararg is not None: + return node.args.vararg + if len(node.args.kwonlyargs) > 0: + return node.args.kwonlyargs[0] + if node.args.kwarg is not None: + return node.args.kwarg + return None + + def process_arg(param: Optional[ast.arg]): + if param is None: + return + r = create_range_from_ast_node(arg) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(param.arg) + new_variable = VariableDeclaration(param.arg, qualified_name, FilePosition(self.current_file, r)) + if param.arg == 'self' and is_method and get_first_param() is param: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + elif is_init: + init_var_placeholder_type = InitVariablePlaceholderType(param.arg) + new_variable.type.add(init_var_placeholder_type) + self.scope_manager.get_current_class_scope().init_placeholders.append(init_var_placeholder_type) + else: + new_variable.type.add(VariablePlaceholderType(param.arg)) + self.scope_manager.append_variable_to_current_scope(new_variable) + + is_init = self.scope_manager.is_current_scope_init() + is_method = self.scope_manager.is_current_scope_method() + for arg in node.args.args: # simple arguments + process_arg(arg) + process_arg(node.args.vararg) # *arg + for arg in node.args.kwonlyargs: # arguments between vararg and kwarg + process_arg(arg) + process_arg(node.args.kwarg) # **arg + for default in node.args.defaults: # last N args default value + self.visit(default) + for default in node.args.kw_defaults: # default values of kwonlyargs (all) + self.visit(default) + for arg in node.args.posonlyargs: # arguments before / "parameter" (after -> args) + process_arg(arg) + + for decorator in node.decorator_list: + self.visit(decorator) + + if node.returns is not None: + self.visit(node.returns) + + for stmt in node.body: + self.visit(stmt) + + def visit_Return(self, node: ast.Return) -> Any: + assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) + if node.value is not None: + types = self.type_deduction.deduct_type(node, self.preprocessed_file) + if len(types) == 0: + self.current_function_declaration[-1].type.add(VariablePlaceholderType('')) + else: + self.current_function_declaration[-1].type.update(types) + self.generic_visit(node) + + def visit_Yield(self, node: ast.Yield) -> Any: + assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) + # TODO: ~Return + self.current_function_declaration[-1].type.update( + self.type_deduction.deduct_type(node, self.preprocessed_file)) + self.generic_visit(node) + + def visit_YieldFrom(self, node: ast.YieldFrom) -> Any: + assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) + # TODO: ~Return + self.current_function_declaration[-1].type.update( + self.type_deduction.deduct_type(node, self.preprocessed_file)) + self.generic_visit(node) + + def visit_NamedExpr(self, node: ast.NamedExpr) -> Any: + fh = None + if isinstance(node.target, ast.Attribute): + fh = MemberAccessCollector(node.target) + name = fh.call_list[0].name + elif isinstance(node.target, ast.Name): + name = node.target.id + elif isinstance(node.target, (ast.Subscript, ast.Starred, ast.List, ast.Tuple)): + return + else: + assert False + + r = create_range_from_ast_node(node.target) + types = self.type_deduction.deduct_type(node.value, self.preprocessed_file) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) + if self.is_static_variable(node.target): + new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + else: + new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + if self.scope_manager.is_current_scope_init() and fh is not None and len(fh.call_list) == 2 and \ + fh.call_list[-1].name == 'self': + if not self.scope_manager.is_declaration_in_current_class(new_variable.name): + self.scope_manager.append_variable_to_current_class_scope(new_variable) + self.scope_manager.get_current_class_declaration().attributes.append(new_variable) + elif isinstance(node.target, ast.Name): + if self.scope_manager.is_inside_conditional_scope() and \ + not isinstance(declaration := self.scope_manager.get_declaration(name), PlaceholderType): + declaration.type.update(new_variable.type) + else: + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + print('') + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> Any: + for target in node.targets: + mac = None + if isinstance(target, ast.Attribute): + mac = MemberAccessCollector(target) + name = mac.call_list[0].name + elif isinstance(target, ast.Name): + name = target.id + elif isinstance(target, (ast.Subscript, ast.Starred, ast.List, ast.Tuple)): + return + else: + assert False + + r = create_range_from_ast_node(target) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) + types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(node.value, self.preprocessed_file)) + if self.is_static_variable(target): + new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + else: + new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + if self.scope_manager.is_current_scope_init() and mac is not None and len(mac.call_list) == 2 and \ + mac.call_list[-1].name == 'self': + if not self.scope_manager.is_declaration_in_current_class(new_variable.name): + self.scope_manager.append_variable_to_current_class_scope(new_variable) + self.scope_manager.get_current_class_declaration().attributes.append(new_variable) + elif isinstance(target, ast.Name): + if self.scope_manager.is_inside_conditional_scope() and \ + not isinstance(declaration := self.scope_manager.get_declaration(name), PlaceholderType): + declaration.type.update(new_variable.type) + else: + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + print('') + + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> Any: + fh = None + if isinstance(node.target, ast.Attribute): + fh = MemberAccessCollector(node.target) + name = fh.call_list[0].name + elif isinstance(node.target, ast.Name): + name = node.target.id + elif isinstance(node.target, (ast.Subscript, ast.Starred, ast.List, ast.Tuple)): + return + else: + assert False + + r = create_range_from_ast_node(node.target) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) + types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(node.value, self.preprocessed_file)) + if self.is_static_variable(node.target): + new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), types) + else: + new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), types) + if self.scope_manager.is_current_scope_init() and fh is not None and \ + len(fh.call_list) == 2 and fh.call_list[-1].name == 'self': + if not self.scope_manager.is_declaration_in_current_class(new_variable.name): + self.scope_manager.append_variable_to_current_class_scope(new_variable) # TODO: use type of self + elif isinstance(node.target, ast.Name): + if not self.scope_manager.is_declaration_in_current_scope(new_variable.name): + self.scope_manager.append_variable_to_current_scope(new_variable) # TODO: not only in last scope + + def visit_For(self, node: ast.For) -> Any: + self.visit_common_for(node) + + def visit_AsyncFor(self, node: ast.AsyncFor) -> Any: + self.visit_common_for(node) + + def visit_common_for(self, node: Union[ast.For, ast.AsyncFor]): + def action(): + if not isinstance(node.target, ast.Tuple) and \ + not self.scope_manager.is_declaration_in_current_scope(node.target.id): + r = create_range_from_ast_node(node.target) + types = self.type_deduction.deduct_type(node.iter, self.preprocessed_file) + qn = self.scope_manager.get_qualified_name_from_current_scope(node.target.id) + new_variable = VariableDeclaration(node.target.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(node.target, ast.Tuple): + for var in node.target.elts: + r = create_range_from_ast_node(var) + # TODO: type + types = set() + # self.type_deduction.deduct_type(node.iter, self.preprocessed_file.class_collector) + qn = self.scope_manager.get_qualified_name_from_current_scope(var.id) + new_variable = VariableDeclaration(var.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + # types = self.type_deduction.deduct_type(...) + # get variable + # add types + pass + self.generic_visit(node) + + self.scope_manager.with_scope(LoopScope(), action) + + def visit_With(self, node: ast.With) -> Any: + self.common_with_visit(node) + + def visit_AsyncWith(self, node: ast.AsyncWith) -> Any: + self.common_with_visit(node) + + def common_with_visit(self, node: Union[ast.With, ast.AsyncWith]) -> Any: + def action(): + for with_item in node.items: + if with_item.optional_vars is not None: + r = create_range_from_ast_node(with_item.optional_vars) + qn = self.scope_manager.get_qualified_name_from_current_scope(with_item.optional_vars.id) + types = self.type_deduction.deduct_type(with_item.context_expr, self.preprocessed_file) + new_variable = VariableDeclaration(with_item.optional_vars.id, qn, + FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(LoopScope(), action) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> Any: + def action(): + if node.name is not None: + r = create_range_from_ast_node(node) + types = self.type_deduction.deduct_type(node.type, self.preprocessed_file) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name) + new_variable = VariableDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), types) + self.scope_manager.append_exception_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(ExceptionScope(), action) + + def visit_Global(self, node: ast.Global) -> Any: + r = create_range_from_ast_node(node) + for name in node.names: + reference = self.scope_manager.get_global_variable(name) + if reference is None: + continue + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) + self.scope_manager.append_variable_to_current_scope( + GlobalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) + if reference is not None: + reference.usages.append(Usage(name, FilePosition(self.current_file, r))) + + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_ListComp(self, node: ast.ListComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_SetComp(self, node: ast.SetComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_DictComp(self, node: ast.DictComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_Lambda(self, node: ast.Lambda) -> Any: + def action(): + for var in node.args.args: + r = create_range_from_ast_node(var) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(var.arg) + new_variable = VariableDeclaration(var.arg, qualified_name, FilePosition(self.current_file, r)) + self.scope_manager.append_variable_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(LambdaScope(), action) + + def visit_comprehension(self, node: ast.comprehension) -> Any: + if isinstance(node.target, ast.Name): + r = create_range_from_ast_node(node.target) + qn = self.scope_manager.get_qualified_name_from_current_scope(node.target.id) + types = self.type_deduction.deduct_type(node.iter, self.preprocessed_file) + new_variable = VariableDeclaration(node.target.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(node.target, ast.Tuple): + for var in node.target.elts: + r = create_range_from_ast_node(var) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(var.id) + types = self.type_deduction.deduct_type(node.iter, self.preprocessed_file) + new_variable = VariableDeclaration(var.id, qualified_name, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + assert False, 'Comprehension target type not handled: ' + type(node.target) + self.generic_visit(node) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> Any: + r = create_range_from_ast_node(node) + for name in node.names: + reference = self.scope_manager.get_nonlocal_variable(name) + if reference is None: + continue + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) + self.scope_manager.append_variable_to_current_scope( + NonlocalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) + if reference is not None: + reference.usages.append(Usage(name, FilePosition(self.current_file, r))) + + def visit_Name(self, node: ast.Name) -> Any: + if isinstance(node.ctx, ast.Store): + var = self.get_declaration(node) + if var is not None: + other = self.create_var_usage(node.id, node) + if not list(var)[0].is_same_usage(other): + self.append_variable_usage(node.id, node) + else: + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.id) + self.scope_manager.append_variable_to_current_scope( + VariableDeclaration(node.id, qualified_name, FilePosition(self.current_file, r))) + assert False, 'Assignment target, was not handled at ' + str(r) + elif isinstance(node.ctx, (ast.Load, ast.Del)): + self.append_variable_usage(node.id, node) + else: + assert False, "Unknown context" + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> Any: + self.append_variable_usage(node.attr, node) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> Any: + types = self.type_deduction.deduct_type(node, self.preprocessed_file) + if len(types) == 0: + print('ph') + for t in types: + from my_ast.type_data import PlaceholderType + if isinstance(t, PlaceholderType): + print('ph') + for t in self.type_deduction.get_current_type(types): + from my_ast.type_data import PlaceholderType + if isinstance(t, PlaceholderType): + print('ph') + + self.generic_visit(node.func) + + for arg in node.args: + self.visit(arg) + for keyword in node.keywords: + self.visit(keyword.value) + + self.append_function_usage(node) + + def visit_Delete(self, node: ast.Delete) -> Any: + # TODO: remove variable (if not -> only runtime error) + self.generic_visit(node) + + def visit_If(self, node: ast.If) -> Any: + # TODO: can be recursive (eg. if..elif..elif..else..), is it a problem? + self.scope_manager.with_scope(ConditionalScope(), lambda: self.generic_visit(node)) + + @staticmethod + def get_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: + for var in scope.declarations: + if var.name == name: + return var + return None + + @staticmethod + def is_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return True + return False + + @staticmethod + def is_variable_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return isinstance(var, VariableDeclaration) + return False + + @staticmethod + def is_function_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return isinstance(var, FunctionDeclaration) + return False + + @staticmethod + def is_class_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return isinstance(var, ClassDeclaration) + return False + + def append_variable_usage(self, name: str, node: (ast.Name, ast.Attribute, ast.Subscript)) -> None: + usage = self.create_var_usage(name, node) + variable_declarations = self.get_declaration(node) + for var in variable_declarations: + if var is not None and not isinstance(var, PlaceholderType): + var.usages.append(usage) + logger.debug('Var Usage: ' + str(usage) + ' ' + var.get_type_repr()) + return + # TODO: annotations -> assert + assert True, "Variable not found: " + usage.name + " (usage: line - " + str(usage.file_position) + ")" + + def append_function_usage(self, node: ast.Call) -> None: + usage = self.create_function_usage(node) + mac = self.member_access_collector_type(node) + is_method_call = (len(mac.call_list) == 2 and + isinstance(mac.call_list[1], MemberAccessCollector.AttributeData) and + mac.call_list[1].name == 'self') + if not (len(mac.call_list) == 1 or is_method_call): + # functions from another module and modules + # return # TODO: handle these cases (type deduction) + pass + function_declarations = self.get_declaration(node) + for func in function_declarations: + if func is not None and not isinstance(func, PlaceholderType): + func.usages.append(usage) + logger.debug('Func Usage: ' + str(usage) + ' ' + func.get_type_repr()) + return + if self.is_builtin(usage.name): + self.append_builtin_usage(usage) + return + # TODO: imported functions/methods from libraries not in the project (builtins!) + assert True, "Function not found: " + usage.name + " (usage: start position - " + str(usage.file_position) + ")" + + def is_builtin(self, name: str) -> bool: + return get_built_in_function(name) is not None + + def append_builtin_usage(self, usage: Usage): + bif = get_built_in_function(usage.name) + if bif is not None: + bif.usages.append(usage) + + def get_declaration(self, node: (ast.Name, ast.Attribute, ast.Call, ast.Subscript)) -> Optional[Set[Declaration]]: + if isinstance(node, ast.Name): + declaration = self.scope_manager.get_declaration(node.id) + if declaration is None: + return None + else: + return {declaration} + + mac = MemberAccessCollector(node) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and len(mac.call_list) == 1: + declaration = self.scope_manager.get_declaration(node.func.id) + if declaration is None: + return None + else: + return {declaration} + elif isinstance(node, ast.Call) and len(mac.call_list) == 1 and not isinstance(node.func, ast.Subscript): + print("NEED FIX: () operator called on lambda, operator or await") + + mac = MemberAccessCollector(node) + last = mac.call_list.pop(0) + types = self.type_deduction.get_member_access_type(mac) + declarations = set() + + for d in types: + if isinstance(d, ModuleVariableDeclaration): + if isinstance(d.imported_module, GlobalScope): + declaration = self.scope_manager.get_declaration_from_scope(last.name, d.imported_module) + if declaration is not None: + declarations.add(declaration) + elif isinstance(d.imported_module, PreprocessedFile): + pass + elif d.imported_module is None: + pass + else: + assert False + elif isinstance(d, (VariableDeclaration, FunctionDeclaration)): + if isinstance(d, (TypeVariable, FunctionVariable)): + current_types = self.type_deduction.get_current_type({d.reference}) + else: + current_types = self.type_deduction.get_current_type(d.type) + for t in current_types: + if isinstance(t, ClassDeclaration): + if t is self.scope_manager.get_current_class_declaration(): + declarations.add(self.scope_manager.get_declaration(last.name)) + elif isinstance(last, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for m in t.attributes: + if m.name == last.name: + declarations.add(m) + for m in t.static_attributes: + if m.name == last.name: + declarations.add(m) + elif isinstance(last, MemberAccessCollector.MethodData): + for m in t.methods: + if m.name == last.name: + declarations.add(m) + for m in t.static_methods: + if m.name == last.name: + declarations.add(m) + else: + print("NEED FIX: BuiltIn, InitVariablePlaceholderType, VariablePlaceholderType") + elif isinstance(d, PlaceholderType): + pass + else: + assert False + + return declarations + + # TODO: change PlaceholderType-s, to actual type + # TODO: declaration is None -> what if not declared in global scope (eg. class member) - pass preprocessed 'scope'? + def post_process(self) -> None: + def post_process_variable(var: PreprocessedVariable) -> None: + var_declaration = self.scope_manager.get_global_scope().get_variable_by_name(var.name) + if var_declaration is None: + return + var_declaration.usages.extend(var.usages) + for type_usage in var.type_usages: + type_usage.type.update(var_declaration.type) + + def post_process_function(func: PreprocessedFunction) -> None: + func_declaration = self.scope_manager.get_global_scope().get_function_by_name(func.name) + if func_declaration is None: + return + func_declaration.usages.extend(func.usages) + for type_usage in func.type_usages: + type_usage.type.update(func_declaration.type) + + def post_process_class(cl: PreprocessedClass) -> None: + for cv in cl.attributes: + post_process_variable(cv) + for cf in cl.methods: + post_process_function(cf) + for cc in cl.classes: + post_process_class(cc) + + class_declaration = self.scope_manager.get_global_scope().get_class_by_name(cl.name) + if class_declaration is None: + return + for type_usage in cl.type_usages: + type_usage.type.update(class_declaration.type) + + for v in self.preprocessed_file.preprocessed_variables: + post_process_variable(v) + for f in self.preprocessed_file.preprocessed_functions: + post_process_function(f) + for c in self.preprocessed_file.class_collector.classes: + post_process_class(c) + + def create_var_usage(self, name: str, param: ast.expr) -> Usage: + r = create_range_from_ast_node(param) + return Usage(name, FilePosition(self.current_file, r)) + + def create_function_usage(self, func: ast.Call) -> Usage: + name = '' + h = ['func'] + while True: + if eval('hasattr(' + '.'.join(h) + ', "value")'): + if eval('isinstance(' + ".".join(h) + ', str)'): + break + else: + h.append('value') + elif eval('hasattr(' + '.'.join(h) + ', "func")'): + h.append('func') + else: + break + + n = [] + for i in range(len(h), 1, -1): + if eval('hasattr(' + '.'.join(h[:i]) + ', "id")'): + id_attr = "id" + elif eval('hasattr(' + '.'.join(h[:i]) + ', "attr")'): + id_attr = "attr" + else: + continue + n.append(eval("getattr(" + '.'.join(h[:i]) + ", '" + id_attr + "')")) + + # asdf = '.'.join(n) + + mac = MemberAccessCollector(func) + + if hasattr(func.func, 'id'): + name = func.func.id + elif hasattr(func.func, 'attr'): + name = func.func.attr + else: + print("NEED FIX: usage when () operator called on return value") + assert True # call on return value (eg. f()(), g[i]()) + + r = create_range_from_ast_node(func) + return Usage(name, FilePosition(self.current_file, r)) + + @staticmethod + def is_static_method(node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> bool: + for decorator in node.decorator_list: + if isinstance(decorator, ast.Name) and decorator.id == 'staticmethod': + return True + return False + + def is_static_variable(self, node: ast.AST) -> bool: + return isinstance(self.scope_manager.get_current_scope(), ClassScope) and isinstance(node, ast.Name) diff --git a/plugins/python/parser/src/my_ast/symbol_collector_interface.py b/plugins/python/parser/src/my_ast/symbol_collector_interface.py new file mode 100644 index 000000000..0cc617d5e --- /dev/null +++ b/plugins/python/parser/src/my_ast/symbol_collector_interface.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from typing import Dict + +from my_ast.base_data import ImportedDeclaration +from my_ast.import_finder import ImportFinder +from my_ast.member_access_collector import MemberAccessCollector +from my_ast.preprocessed_file import PreprocessedFile +from my_ast.scope_manager import ScopeManager + + +class ISymbolCollector(ABC): + @abstractmethod + def collect_symbols(self): + pass + + +class SymbolCollectorBase(ISymbolCollector, ABC): + def __init__(self, preprocessed_file: PreprocessedFile, import_finder: ImportFinder): + self.preprocessed_file: PreprocessedFile = preprocessed_file + self.import_finder: ImportFinder = import_finder + self.member_access_collector_type = MemberAccessCollector + self.imported_declaration_scope_map: Dict[ImportedDeclaration, ScopeManager] = {} + + +class IFunctionSymbolCollector(ISymbolCollector, ABC): + def __init__(self): + self.function = None diff --git a/plugins/python/parser/src/my_ast/symbol_finder.py b/plugins/python/parser/src/my_ast/symbol_finder.py new file mode 100644 index 000000000..23189bd15 --- /dev/null +++ b/plugins/python/parser/src/my_ast/symbol_finder.py @@ -0,0 +1,10 @@ +from typing import Optional +from abc import ABC, abstractmethod + +from my_ast.base_data import Declaration + + +class SymbolFinder(ABC): + @abstractmethod + def get_declaration(self, name: str) -> Optional[Declaration]: + pass diff --git a/plugins/python/parser/src/my_ast/test.log b/plugins/python/parser/src/my_ast/test.log new file mode 100644 index 000000000..eae8aac0b --- /dev/null +++ b/plugins/python/parser/src/my_ast/test.log @@ -0,0 +1,27 @@ + +FILE: __init__ +========================================= +Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] + +FILE: asdf +========================================= + +FILE: qwer +========================================= +Func: g (line: 1, column: 0, length: 1) {global - } [] + +FILE: asdf +========================================= +Func: g (line: 0, column: 0, length: 0) {global - } [] +Func: f (line: 4, column: 0, length: 1) {global - } [] +Var: a (line: 4, column: 0, length: 1) {function - f} [Placeholder] +Var: b (line: 4, column: 0, length: 1) {function - f} [Placeholder] +Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] +Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] +Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] +Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] +Func Usage: g (line: 7, column: 4, length: 1) [] + +FILE: __init__ +========================================= +Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] diff --git a/plugins/python/parser/src/my_ast/type_data.py b/plugins/python/parser/src/my_ast/type_data.py new file mode 100644 index 000000000..7e2956d26 --- /dev/null +++ b/plugins/python/parser/src/my_ast/type_data.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod + + +class DeclarationType(ABC): + def __init__(self, name: str, qualified_name: str): + self.name = name + self.qualified_name = qualified_name + + @abstractmethod + def get_type_repr(self) -> str: + pass + + @abstractmethod + def __hash__(self): + pass + + @abstractmethod + def __eq__(self, other): + pass + + @abstractmethod + def __ne__(self, other): + pass + + +class PlaceholderType(DeclarationType): + def __init__(self, name: str): + DeclarationType.__init__(self, name, "") + + def get_type_repr(self) -> str: + return 'Placeholder' + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + +class VariablePlaceholderType(PlaceholderType): + pass + + +class InitVariablePlaceholderType(VariablePlaceholderType): + pass + + +class FunctionPlaceholderType(PlaceholderType): + pass diff --git a/plugins/python/parser/src/my_ast/type_deduction.py b/plugins/python/parser/src/my_ast/type_deduction.py new file mode 100644 index 000000000..a1d9569b0 --- /dev/null +++ b/plugins/python/parser/src/my_ast/type_deduction.py @@ -0,0 +1,507 @@ +import ast +import typing +from functools import singledispatchmethod +from typing import Optional, Any, Union + +from my_ast import built_in_types +from my_ast.base_data import Declaration, ImportedDeclaration, Usage +from my_ast.built_in_functions import get_built_in_function +from my_ast.built_in_operators import get_built_in_operator +from my_ast.built_in_types import Boolean, Dictionary, Set, Tuple, String, Integer, Float, Bytes, \ + EllipsisType, NoneType, Complex, RangeType, BuiltIn, Type, GenericBuiltInType +from my_ast.class_data import ClassDeclaration +from my_ast.common.file_position import FilePosition +from my_ast.common.hashable_list import OrderedHashableList +from my_ast.function_data import FunctionDeclaration +from my_ast.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from my_ast.member_access_collector import MemberAccessCollector, MemberAccessCollectorIterator +from my_ast.preprocessed_file import PreprocessedFile +from my_ast.scope_manager import ScopeManager +from my_ast.symbol_collector_interface import SymbolCollectorBase +from my_ast.type_data import DeclarationType, PlaceholderType, VariablePlaceholderType, FunctionPlaceholderType,\ + InitVariablePlaceholderType +from my_ast.variable_data import VariableDeclaration, FunctionVariable, TypeVariable + + +class TypeDeduction: + def __init__(self, symbol_collector: SymbolCollectorBase, scope_manager: ScopeManager, + function_symbol_collector_factory: FunctionSymbolCollectorFactory): + self.scope_manager: ScopeManager = scope_manager + self.preprocessed_file: Optional[PreprocessedFile] = None + self.symbol_collector: SymbolCollectorBase = symbol_collector + self.function_symbol_collector_factory: FunctionSymbolCollectorFactory = function_symbol_collector_factory + + def deduct_type(self, node: ast.AST, preprocessed_file: PreprocessedFile) -> typing.Set: + self.preprocessed_file = preprocessed_file + types = self.get_type(node) + if not isinstance(types, typing.Set): + return {types} + return types + + def get_current_type(self, types: typing.Set[DeclarationType])\ + -> typing.Set[Union[ClassDeclaration, BuiltIn, FunctionVariable, TypeVariable, PlaceholderType]]: + def get_current_type_impl(declaration_type: DeclarationType)\ + -> typing.Set[Union[ClassDeclaration, BuiltIn, FunctionVariable, TypeVariable, PlaceholderType]]: + if isinstance(declaration_type, PlaceholderType): + if isinstance(declaration_type, InitVariablePlaceholderType): + return {declaration_type} + declaration = self.scope_manager.get_declaration(declaration_type.name) + if isinstance(declaration, PlaceholderType) and declaration_type.name == declaration.name: + return {declaration} + return self.get_current_type({declaration}) + elif isinstance(declaration_type, FunctionDeclaration): + return self.get_current_type(declaration_type.type) + elif isinstance(declaration_type, (ClassDeclaration, BuiltIn, FunctionVariable, TypeVariable)): + return {declaration_type} + elif isinstance(declaration_type, VariableDeclaration): + return declaration_type.type # VariableDeclaration.type should only be ClassDeclaration and BuiltIn + else: + assert False + + current_types = set() + for t in types: + if t is None: + continue + current_types.update(get_current_type_impl(t)) + return current_types + + @singledispatchmethod + def get_type(self, node: ast.AST) -> Any: + assert False, "Unknown type: " + str(type(node)) + + @get_type.register + def _(self, node: ast.BoolOp): + return Boolean() + + @get_type.register + def _(self, node: ast.NamedExpr): # (x := y) eg. if (y := f(x)) is not None: ... + pass + + @get_type.register + def _(self, node: ast.BinOp): + left_operand_type = self.get_current_type(self.deduct_type(node.left, self.preprocessed_file)) + bio = get_built_in_operator(node.op) + assert bio is not None + types = set() + for t in left_operand_type: + if isinstance(t, (BuiltIn, PlaceholderType)): + types.add(t) # TODO: handle numbers + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: check this (eg. base class is BuiltIn) + types.add(t) + else: + print("???") + return types # type(left_operand_type.op) + + @get_type.register + def _(self, node: ast.UnaryOp): + operand = self.get_current_type(self.deduct_type(node.operand, self.preprocessed_file)) + bio = get_built_in_operator(node.op) + assert bio is not None + types = set() + for t in operand: + if isinstance(t, BuiltIn): + types.add(t) + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + assert False # TODO: is this a valid case? (operator on custom class without override it) + else: + print("???") + return types + + @get_type.register + def _(self, node: ast.Lambda): + # add node.args to tmp_scope + return self.get_type(node.body) + + @get_type.register + def _(self, node: ast.IfExp): + return {self.get_type(node.body), self.get_type(node.orelse)} + + @get_type.register + def _(self, node: ast.Dict): + return Dictionary(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.Set): + return Set(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.ListComp): + return built_in_types.List() # generic types? + + @get_type.register + def _(self, node: ast.SetComp): + return Set() # generic types? + + @get_type.register + def _(self, node: ast.DictComp): + return Dictionary() # generic types? + + @get_type.register + def _(self, node: ast.GeneratorExp): + pass # tmp_var := type(iter) -> type(target) + + @get_type.register + def _(self, node: ast.Await): + return self.get_type(node.expr) + + @get_type.register + def _(self, node: ast.Compare): + # TODO: handle PlaceHolderType + left_values = self.deduct_type(node.left, self.preprocessed_file) + types = set() + assert len(node.comparators) == len(node.ops) + for i in range(0, len(node.comparators)): + bio = get_built_in_operator(node.ops[i]) + assert bio is not None + left_operand_type = self.get_current_type(left_values) + for t in left_operand_type: + if isinstance(t, BuiltIn): + types.add(Boolean()) + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: check this (eg. base class is BuiltIn) + types.add(t) + else: + print("???") + left_values = types + return left_values + + @get_type.register + def _(self, node: ast.Call): + if hasattr(node.func, 'id'): + # TODO: check if builtin class ctor-s are not hidden by other (custom) functions/classes + # TODO: no args check (builtin is not hidden + correct code -> should not be a problem) + built_in_function = get_built_in_function(node.func.id) + if built_in_function is not None: + return built_in_function + elif node.func.id == 'TypeVar': + return Type() + return self.get_member_access_type(MemberAccessCollector(node)) + + @get_type.register + def _(self, node: ast.FormattedValue): # {x} in a joined string + return self.get_type(node.value) # cannot be on right side without f-string? + + @get_type.register + def _(self, node: ast.JoinedStr): # f"... {x} ... {y} ..." + return String() + + @get_type.register + def _(self, node: ast.Constant): + if isinstance(node.value, int): + if isinstance(node.value, bool): + return Boolean() + else: + return Integer() + elif isinstance(node.value, float): + return Float() + elif isinstance(node.value, bytes): + return Bytes() + elif isinstance(node.value, complex): + return Complex() + elif isinstance(node.value, str): + return String() + elif node.value is None: + return NoneType() + elif node.value is Ellipsis: + return EllipsisType() + return set() + + @get_type.register + def _(self, node: ast.Attribute): + return self.get_member_access_type(MemberAccessCollector(node)) + + @get_type.register + def _(self, node: ast.Subscript): + if isinstance(node.slice, ast.Index): + if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) + return Integer() + else: + if isinstance(node.value, ast.Name): + t = self.deduct_type(node.value, self.preprocessed_file) + elif isinstance(node.value, ast.Attribute): + t = self.deduct_type(node.value, self.preprocessed_file) + elif isinstance(node.value, ast.Subscript): + t = self.deduct_type(node.value, self.preprocessed_file) + else: + assert False + return set() # type(node.value->GenericType) + elif isinstance(node.slice, (ast.Slice, ast.ExtSlice)): # ExtSlice -> array type (module arr) + if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) + return RangeType() + else: + return set() # type(node.value)[type(node.value->GenericType)] + return set() # {type(value)...} + + @get_type.register + def _(self, node: ast.Starred): # unpack: * (eg. list, tuple) and ** (eg. dictionary) + return set() # could be same as iterator + + @get_type.register + def _(self, node: ast.Name): + return self.scope_manager.get_declaration(node.id) + + @get_type.register + def _(self, node: ast.List): + return built_in_types.List(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.Tuple): + return Tuple(self.get_element_types(node)) + + @get_type.register(ast.Return) + @get_type.register(ast.Yield) + @get_type.register(ast.YieldFrom) + def _(self, node): + return self.get_type_of_function(node) + + @get_type.register + def _(self, node: ast.Expr): + return self.get_type(node.value) + + def get_element_types(self, node: ast.AST): + element_types = set() + if isinstance(node, ast.Dict): + elements = getattr(node, 'values') + else: + assert hasattr(node, 'elts') + elements = getattr(node, 'elts') + for elem in elements: + elem_type = self.get_type(elem) + if elem_type is not None: + types = self.get_type(elem) + if isinstance(types, typing.Set): + element_types.update(types) + else: + element_types.add(types) + return element_types + + def get_member_access_type(self, mac: MemberAccessCollector): + if len(mac.call_list) == 0: + return set() + iterator = MemberAccessCollectorIterator(mac) + declaration = self.scope_manager.get_declaration_from_member_access(iterator) + if declaration is None or isinstance(declaration, PlaceholderType): + if iterator.is_iteration_started(): + a = iterator.get_current() + else: + a = iterator.get_first() + + # TODO: is empty FilePosition correct? + if isinstance(a, MemberAccessCollector.MethodData): + b = [x for x in self.preprocessed_file.preprocessed_functions if x.name == a.name] + c = [x for x in self.preprocessed_file.class_collector.classes if x.name == a.name] + for i in b: + i.usages.append(Usage(a.name, FilePosition.get_empty_file_position())) + for i in c: + i.usages.append(Usage(a.name, FilePosition.get_empty_file_position())) + elif isinstance(a, (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + b = [x for x in self.preprocessed_file.preprocessed_variables if x.name == a.name] + for i in b: + i.usages.append(Usage(a.name, FilePosition.get_empty_file_position())) + declarations = {declaration} + else: + declarations = {declaration} + if iterator.is_iteration_over() or not iterator.is_iteration_started(): + if not any(isinstance(x, PlaceholderType) for x in declarations) and \ + any(any(isinstance(x, PlaceholderType) for x in y.type) for y in declarations): + declarations = self.get_current_type_of_placeholder_function(declarations, mac.call_list[-1]) + return declarations + prev_member = iterator.get_current() + next(iterator) + for member in mac.call_list[iterator.index::-1]: + if len(declarations) == 0 or all(x is None for x in declarations): + return set() + elif all(isinstance(x, PlaceholderType) for x in declarations): + return set() + declarations = self.get_member_declarations(declarations, prev_member, member) + prev_member = member + return declarations + + def get_member_declarations(self, declarations: typing.Set[DeclarationType], + current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + member_declarations = set() + for declaration in declarations: + for declaration_type in self.get_current_type(declaration.type): + if isinstance(declaration_type, PlaceholderType) and isinstance(declaration, FunctionDeclaration): + d = self.get_current_type_of_placeholder_function({declaration}, current_member) + for dt in d: + for declaration_current_type in self.get_possible_types(dt): + if isinstance(declaration_current_type, BuiltIn): + continue + elif isinstance(declaration_current_type, ClassDeclaration): + member_declarations.update( + self.get_member(declaration_current_type, current_member, next_member)) + elif isinstance(declaration_current_type, VariablePlaceholderType): + continue # TODO: handle this (eg. function parameters) + else: + assert False + # member_declarations.update(d) + continue # TODO: try to get current value of function with the arguments from 'mac' + for declaration_current_type in self.get_possible_types(declaration_type): + if isinstance(declaration_current_type, BuiltIn): + self.get_member_of_builtin(declaration_current_type, current_member, next_member) + elif isinstance(declaration_current_type, ClassDeclaration): + member_declarations.update(self.get_member( + declaration_current_type, current_member, next_member)) + elif isinstance(declaration_current_type, VariablePlaceholderType): + continue # TODO: handle this (eg. function parameters) + elif isinstance(declaration_current_type, InitVariablePlaceholderType): + continue + else: + assert False + return member_declarations + + def get_member(self, declaration: Declaration, current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + types = self.get_possible_types(declaration) + declarations = set() + for t in types: + if isinstance(t, ClassDeclaration): + if isinstance(next_member, (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for attr in reversed(t.attributes): + if attr.name == next_member.name: + declarations.add(attr) + for attr in reversed(t.static_attributes): + if attr.name == next_member.name: + declarations.add(attr) + elif isinstance(next_member, MemberAccessCollector.MethodData): + for method in reversed(t.methods): + if method.name == next_member.name: + declarations.add(method) + for method in reversed(t.static_methods): + if method.name == next_member.name: + declarations.add(method) + elif isinstance(next_member, MemberAccessCollector.SubscriptData): + # TODO: subscript elements type -> init/call method + pass + elif isinstance(next_member, MemberAccessCollector.LambdaData): + # TODO: member.node type -> init/call method + pass + elif isinstance(next_member, MemberAccessCollector.OperatorData): + # TODO: member.node type -> init/call method + pass + elif isinstance(t, BuiltIn): + continue + else: + assert False + return declarations + + def get_member_of_builtin(self, builtin: BuiltIn, current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + if isinstance(builtin, GenericBuiltInType) and isinstance(current_member, MemberAccessCollector.SubscriptData): + return builtin.types + return set() + + # evaluate declaration to get type + def get_possible_types(self, declaration: DeclarationType) -> typing.Set[Union[ClassDeclaration, BuiltIn]]: + types = set() + # TODO: handle BuiltIn-s (Module!) + if isinstance(declaration, BuiltIn): + return {declaration} + elif isinstance(declaration, Declaration): + for t in declaration.type: + if isinstance(t, (ClassDeclaration, BuiltIn, PlaceholderType)): + types.add(t) + elif isinstance(t, (VariableDeclaration, FunctionDeclaration)): + types.update(self.get_possible_types(t)) + else: + assert False, "Unknown type: " + str(type(t)) + elif isinstance(declaration, PlaceholderType): + # TODO: get current value of function or variable + return self.get_current_type({self.scope_manager.get_declaration(declaration.name)}) + else: + assert False, "Unknown declaration type: " + str(type(declaration)) + return types + + def get_declaration(self, declaration: MemberAccessCollector.MemberData) \ + -> Optional[Union[VariableDeclaration, FunctionDeclaration]]: + # TODO: handle global, nonlocal and import declarations + return self.scope_manager.get_declaration(declaration.name) + + def get_current_type_of_placeholder_function(self, declarations: typing.Set, + member: MemberAccessCollector.MemberData) \ + -> typing.Set[DeclarationType]: + types = set() + for declaration in declarations: + arguments = OrderedHashableList() + if isinstance(member, MemberAccessCollector.MethodData): + for arg in member.arguments: + arguments.append(self.deduct_type(arg, self.preprocessed_file)) + declaration_type = self.scope_manager.placeholder_function_declaration_cache. \ + get_functions_return_type(declarations, arguments) + if declaration_type is not None and len(declaration_type) > 0: + types.update(declaration_type) + else: + if isinstance(declaration, ImportedDeclaration): + scope_manager = self.symbol_collector.imported_declaration_scope_map[declaration] + func_def = scope_manager.placeholder_function_declaration_cache. \ + get_function_def(declaration.imported_declaration) + else: + func_def = self.scope_manager.placeholder_function_declaration_cache.get_function_def(declaration) + # assert func_def is not None # VariableDeclaration? + if func_def is None: + continue + # magic + if len(arguments) == 0: + sc = self.function_symbol_collector_factory.get_function_symbol_collector( + self.symbol_collector, func_def, []) + sc.collect_symbols() + types.update(sc.function.type) + else: + for args in self.get_argument_combinations(arguments): + sc = self.function_symbol_collector_factory.get_function_symbol_collector( + self.symbol_collector, func_def, args) + sc.collect_symbols() + types.update(sc.function.type) + return types + + def get_argument_combinations(self, arguments: OrderedHashableList[typing.Set[DeclarationType]]) \ + -> typing.Iterator[OrderedHashableList[DeclarationType]]: + if len(arguments) == 0: + return OrderedHashableList() + for argument in arguments[0]: + head_argument = OrderedHashableList([argument]) + if len(arguments) == 1: + yield head_argument + else: + for tail_arguments in self.get_argument_combinations(arguments[1::]): + yield head_argument + tail_arguments + + def get_type_of_function(self, node: Union[ast.Return, ast.Yield, ast.YieldFrom]): + if node.value is None: + return set() + + types = self.get_type(node.value) + if types is None: + return set() + fixed_types = set() + if isinstance(types, typing.Set): + fixed_types.update(map(lambda x: self.fix_placeholder(x), types)) + else: + fixed_types.add(self.fix_placeholder(types)) + return fixed_types + + def fix_placeholder(self, declaration_type: DeclarationType): + if any(isinstance(x, PlaceholderType) for x in self.get_current_type({declaration_type})): + if isinstance(declaration_type, VariableDeclaration): + return VariablePlaceholderType(declaration_type.name) + elif isinstance(declaration_type, FunctionDeclaration): + return FunctionPlaceholderType(declaration_type.name) + return declaration_type diff --git a/plugins/python/parser/src/my_ast/variable_data.py b/plugins/python/parser/src/my_ast/variable_data.py new file mode 100644 index 000000000..db4a40fd6 --- /dev/null +++ b/plugins/python/parser/src/my_ast/variable_data.py @@ -0,0 +1,77 @@ +from pathlib import PurePath +from typing import Optional, Set, Union + +from my_ast.built_in_types import Module, BuiltIn +from my_ast.common.file_position import FilePosition +from my_ast.base_data import Declaration, TypeDeclaration, ImportedDeclaration +from my_ast.function_data import FunctionDeclaration +from my_ast.persistence.base_dto import UsageDTO +from my_ast.persistence.variable_dto import VariableDeclarationDTO + + +class VariableDeclaration(Declaration): + # var_type: Optional[Set[Union[cd.ClassDeclaration, BuiltIn]]] - circular import + def __init__(self, name: str, qualified_name: str, pos: FilePosition, declaration_type: Optional[Set] = None): + super().__init__(name, qualified_name, pos, declaration_type) + # self.type: VariableDeclaration and FunctionDeclaration type can change + # (eg. new assignment with different type or redefinition) + + def create_dto(self) -> VariableDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + for t in self.type: + types.add(t.qualified_name) + return VariableDeclarationDTO(self.name, self.qualified_name, self.file_position, types, usages) + + +class StaticVariableDeclaration(VariableDeclaration): + pass + + +class ReferenceVariableDeclaration(VariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: Declaration): + super().__init__(name, qualified_name, pos, reference.type) + self.reference: Declaration = reference + + +class NonlocalVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): + super().__init__(name, qualified_name, pos, reference) + + +class GlobalVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): + super().__init__(name, qualified_name, pos, reference) + + +# TODO: short/long name? (in case of import) +class ModuleVariableDeclaration(VariableDeclaration): + # module: Union[GlobalScope, PreprocessedFile] - circular import + def __init__(self, name: str, location: PurePath, pos: FilePosition, module): + super().__init__(name, "", pos, {Module()}) # TODO: need qualified name? + self.imported_module = module + self.imported_module_location: PurePath = location + + +class ImportedVariableDeclaration(VariableDeclaration, ImportedDeclaration[VariableDeclaration]): + def __init__(self, name: str, pos: FilePosition, var_declaration: VariableDeclaration): + VariableDeclaration.__init__(self, name, "", pos, var_declaration.type) + ImportedDeclaration.__init__(self, var_declaration) + + +class TypeVariable(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, type_ref: TypeDeclaration): + super().__init__(name, qualified_name, pos, type_ref) + + def get_type_repr(self) -> str: + return '[TypeVariable(' + self.reference.name + ')]' + + +class FunctionVariable(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, func_ref: FunctionDeclaration): + super().__init__(name, qualified_name, pos, func_ref) + + def get_type_repr(self) -> str: + return '[FunctionVariable(' + self.reference.name + ')]' diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp new file mode 100644 index 000000000..48e84582c --- /dev/null +++ b/plugins/python/parser/src/pythonparser.cpp @@ -0,0 +1,678 @@ +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cc { +namespace parser{ + +class Persistence +{ +private: + ParserContext& ctx; + +public: + Persistence(ParserContext& ctx_) : ctx(ctx_) {} + + void f() { std::cout << "C++" << std::endl; } + void persistFile(boost::python::object pyFile); + void persistVariable(boost::python::object pyVariable); + void persistFunction(boost::python::object pyFunction); + void persistClass(boost::python::object pyClass); + void persistImport(boost::python::object pyImport); + +private: + boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); +}; + +void Persistence::persistFile(boost::python::object pyFile) +{ + model::FilePtr file = nullptr; + model::BuildSource buildSource; + + boost::python::object path = pyFile.attr("path"); + boost::python::object status = pyFile.attr("parse_status"); + + if(status.is_none()){ + std::cout << "status is None..." << std::endl; + } else if(path.is_none()){ + std::cout << "path is None..." << std::endl; + } else { + buildSource.file = ctx.srcMgr.getFile(boost::python::extract(path)); + std::cout << buildSource.file << std::endl; + switch(boost::python::extract(status)){ + case 0: + buildSource.file->parseStatus = model::File::PSNone; + break; + case 1: + buildSource.file->parseStatus = model::File::PSPartiallyParsed; + break; + case 2: + buildSource.file->parseStatus = model::File::PSFullyParsed; + break; + default: + std::cout << "Unknown status: " << boost::python::extract(status) << std::endl; + } + + model::BuildActionPtr buildAction(new model::BuildAction); + buildAction->command = ""; + buildAction->type = model::BuildAction::Other; + try{ + util::OdbTransaction transaction{ctx.db}; + + transaction([&, this] { + ctx.db->persist(buildAction); + }); + + buildSource.action = buildAction; + + ctx.srcMgr.updateFile(*buildSource.file); + + transaction([&, this] { + ctx.db->persist(buildSource); + }); + } catch(const std::exception& ex){ + std::cout << "Exception: " << ex.what() << " - " << typeid(ex).name() << std::endl; + } + } +// std::cout << std::endl << "START >>>" << std::endl; +// +// try{ +// boost::python::object file_name = pyFile.attr("file_name"); +// +// if(file_name.is_none()){ +// std::cout << "file name is None..." << std::endl; +// } else { +// std::string s = boost::python::extract(file_name); +// std::cout << "File name: " << s << std::endl; +// file.filename = s; +// } +// +// boost::python::object type = pyFile.attr("type"); +// +// if(type.is_none()){ +// std::cout << "type name is None..." << std::endl; +// } else { +// std::string s = boost::python::extract(type); +// std::cout << "Type: " << s << std::endl; +// file.type = s; +// } +// +// boost::python::object path = pyFile.attr("path"); +// +// if(path.is_none()){ +// std::cout << "path is None..." << std::endl; +// } else { +// std::string s = boost::python::extract(path); +// std::cout << "Path: " << s << std::endl; +// +// boost::system::error_code ec; +// boost::filesystem::path p(s); +// std::time_t timestamp = boost::filesystem::last_write_time(p, ec); +// if(ec){ +// timestamp = 0; +// } +// +// file.path = s; +// file.id = util::fnvHash(s); +// file.timestamp = timestamp; +// } +// +// boost::python::object directory = pyFile.attr("parent"); +// +// if(directory.is_none()){ +// std::cout << "directory is None..." << std::endl; +// } else { +// std::string s = boost::python::extract(directory); +// std::cout << "Parent folder: " << s << std::endl; +// +// boost::filesystem::path parentPath = boost::filesystem::path(s); +// +// if (parentPath.native().empty()){ +// file.parent = nullptr; +// } else { +// file.parent = nullptr; // getFile(parentPath.native()); +// } +// } +// +// boost::python::object content_wrapper = pyFile.attr("content"); +// +// if(content_wrapper.is_none()){ +// std::cout << "content_wrapper is None..." << std::endl; +// } else { +// boost::python::object content = content_wrapper.attr("content"); +// +// if(content.is_none()){ +// std::cout << "content is None..." << std::endl; +// } else { +// std::string s = boost::python::extract(content); +// std::cout << "Content:\n" << s << std::endl; +// +// model::FileContentPtr file_content(new model::FileContent); +// +// file_content->hash = util::sha1Hash(s); +// file_content->content = s; +// +// file.content = file_content; +// } +// } +// +// boost::python::object status = pyFile.attr("parse_status"); +// +// if(status.is_none()){ +// std::cout << "status is None..." << std::endl; +// } else { +// int s = boost::python::extract(status); +// std::cout << "Status: " << s << std::endl; +// +// switch(s){ +// case 0: +// file.parseStatus = model::File::PSNone; +// break; +// case 1: +// file.parseStatus = model::File::PSPartiallyParsed; +// break; +// case 2: +// file.parseStatus = model::File::PSFullyParsed; +// break; +// default: +// std::cout << "Unknown status: " << s << std::endl; +// } +// } +// } catch(const boost::python::error_already_set&) { +// std::cout << "error_already_set" << std::endl; +// } catch(const std::exception& ex) { +// std::cout << ex.what() << std::endl; +// } + +// std::cout << "<<< END" << std::endl << std::endl; +} + +void Persistence::persistVariable(boost::python::object pyVariable) +{ + boost::python::object name = pyVariable.attr("name"); + boost::python::object qualifiedName = pyVariable.attr("qualified_name"); + + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyVariable.attr("file_position")); + + // set + boost::python::list types = boost::python::extract(pyVariable.attr("type")); + + boost::python::list usages = boost::python::extract(pyVariable.attr("usages")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + types.is_none() || usages.is_none()){ + return; + } + + util::OdbTransaction transaction{ctx.db}; + + model::PythonAstNodePtr varAstNode(new model::PythonAstNode); + varAstNode->location = fileLoc.get(); + varAstNode->qualifiedName = boost::python::extract(qualifiedName); + varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + varAstNode->astType = model::PythonAstNode::AstType::Declaration; + + transaction([&, this] { + ctx.db->persist(varAstNode); + }); + + model::PythonVariablePtr variable(new model::PythonVariable); +// variable->astNodeId = ? need to insert varAstNode first? + variable->name = boost::python::extract(name); + variable->qualifiedName = boost::python::extract(qualifiedName); + + transaction([&, this] { + ctx.db->persist(variable); + }); + + for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); +// type->symbol = id of variable + + transaction([&, this] { + ctx.db->persist(type); + }); + } + + for(int i = 0; i usageFileLoc = + createFileLocFromPythonFilePosition(usages[i].attr("file_position")); + if(usageFileLoc == boost::none){ + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + transaction([&, this] { + ctx.db->persist(usageAstNode); + }); + } +} + +void Persistence::persistFunction(boost::python::object pyFunction) +{ + boost::python::object name = pyFunction.attr("name"); + boost::python::object qualifiedName = pyFunction.attr("qualified_name"); + + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyFunction.attr("file_position")); + + // set + boost::python::list types = boost::python::extract(pyFunction.attr("type")); + + boost::python::list usages = boost::python::extract(pyFunction.attr("usages")); + + boost::python::object pyDocumentation = pyFunction.attr("documentation"); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + types.is_none() || usages.is_none() || pyDocumentation.is_none()){ + return; + } + + util::OdbTransaction transaction{ctx.db}; + + model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); + funcAstNode->location = fileLoc.get(); + funcAstNode->qualifiedName = boost::python::extract(qualifiedName); + funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + funcAstNode->astType = model::PythonAstNode::AstType::Declaration; + + transaction([&, this] { + ctx.db->persist(funcAstNode); + }); + + model::PythonFunctionPtr function(new model::PythonFunction); +// function->astNodeId = ? need to insert varAstNode first? + function->name = boost::python::extract(name); + function->qualifiedName = boost::python::extract(qualifiedName); + + transaction([&, this] { + ctx.db->persist(function); + }); + + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = boost::python::extract(pyDocumentation); +// documentation->documented = id of function + documentation->documentationKind = model::PythonDocumentation::Function; + + transaction([&, this] { + ctx.db->persist(documentation); + }); + + for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); +// type->symbol = id of function + + transaction([&, this] { + ctx.db->persist(type); + }); + } + + for(int i = 0; i usageFileLoc = + createFileLocFromPythonFilePosition(usages[i].attr("file_position")); + if(usageFileLoc == boost::none){ + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + transaction([&, this] { + ctx.db->persist(usageAstNode); + }); + } +} + +void Persistence::persistClass(boost::python::object pyClass) +{ + boost::python::object name = pyClass.attr("name"); + boost::python::object qualifiedName = pyClass.attr("qualified_name"); + + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyClass.attr("file_position")); + + boost::python::list usages = boost::python::extract(pyClass.attr("usages")); + + boost::python::object pyDocumentation = pyClass.attr("documentation"); + + // set + boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); + + boost::python::object members = pyClass.attr("members"); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + usages.is_none() || pyDocumentation.is_none() || baseClasses.is_none() || members.is_none()){ + return; + } + + util::OdbTransaction transaction{ctx.db}; + + model::PythonAstNodePtr classAstNode(new model::PythonAstNode); + classAstNode->location = fileLoc.get(); + classAstNode->qualifiedName = boost::python::extract(qualifiedName); + classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + classAstNode->astType = model::PythonAstNode::AstType::Declaration; + + transaction([&, this] { + ctx.db->persist(classAstNode); + }); + + model::PythonClassPtr cl(new model::PythonClass); +// function->astNodeId = ? need to insert varAstNode first? + cl->name = boost::python::extract(name); + cl->qualifiedName = boost::python::extract(qualifiedName); + + transaction([&, this] { + ctx.db->persist(cl); + }); + + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = boost::python::extract(pyDocumentation); +// documentation->documented = id of function + documentation->documentationKind = model::PythonDocumentation::Class; + + transaction([&, this] { + ctx.db->persist(documentation); + }); + + for(int i = 0; i usageFileLoc = + createFileLocFromPythonFilePosition(usages[i].attr("file_position")); + if(usageFileLoc == boost::none){ + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + transaction([&, this] { + ctx.db->persist(usageAstNode); + }); + } + + for(int i = 0; iderived = id of cl +// inheritance->base = getPythonEntityIdByQualifiedName(boost::python::extract(baseClasses[i])); + + transaction([&, this] { + ctx.db->persist(inheritance); + }); + } + + boost::python::list methods = boost::python::extract(members.attr("methods")); + boost::python::list staticMethods = boost::python::extract(members.attr("static_methods")); + boost::python::list attributes = boost::python::extract(members.attr("attributes")); + boost::python::list staticAttributes = boost::python::extract(members.attr("static_attributes")); + boost::python::list classes = boost::python::extract(members.attr("classes")); + + if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || + staticAttributes.is_none() || classes.is_none()){ + return; + } + + for(int i = 0; iastNodeId = id of classAstNode +// classMember->memberId = search for function declaration +// classMember->classId = id of cl + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = false; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = id of classAstNode +// classMember->memberId = search for function declaration +// classMember->classId = id of cl + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = true; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = id of classAstNode +// classMember->memberId = search for variable declaration +// classMember->classId = id of cl + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = false; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = id of classAstNode +// classMember->memberId = search for variable declaration +// classMember->classId = id of cl + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = true; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = id of classAstNode +// classMember->memberId = search for class declaration +// classMember->classId = id of cl + classMember->kind = model::PythonClassMember::Class; + classMember->staticMember = false; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } +} + +void Persistence::persistImport(boost::python::object pyImport) +{ + model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); + + boost::python::list importedModules = + boost::python::extract(pyImport.attr("imported_modules")); + + boost::python::dict importedSymbols = + boost::python::extract(pyImport.attr("imported_symbols")); + + if(file == nullptr || importedModules.is_none() || importedSymbols.is_none()){ + return; + } + + util::OdbTransaction transaction{ctx.db}; + + for(int i = 0; i(importedModules[i])); + if(moduleFile == nullptr){ + continue; + } + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->importer = file; + moduleImport->imported = moduleFile; + + transaction([&, this] { + ctx.db->persist(moduleImport); + }); + } + + boost::python::list importDict = importedSymbols.items(); + for(int i = 0; i(importDict[i]); + model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(import[0])); + if(moduleFile == nullptr){ + continue; + } + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->importer = file; + moduleImport->imported = moduleFile; +// moduleImport->importedSymbol = getPythonEntityIdByQualifiedName(boost::python::extract(import[1])); + + transaction([&, this] { + ctx.db->persist(moduleImport); + }); + } +} + +boost::optional Persistence::createFileLocFromPythonFilePosition(boost::python::object filePosition) +{ + if (filePosition.is_none()){ + return boost::none; + } + + boost::python::object filePath = filePosition.attr("file"); + boost::python::object pyRange = filePosition.attr("range"); + + if (filePath.is_none() || pyRange.is_none()){ + return boost::none; + } + + boost::python::object pyStartPosition = pyRange.attr("start_position"); + boost::python::object pyEndPosition = pyRange.attr("end_position"); + + if (pyStartPosition.is_none() || pyEndPosition.is_none()){ + return boost::none; + } + + model::FileLoc fileLoc; + + fileLoc.file = ctx.srcMgr.getFile(boost::python::extract(filePath)); + + model::Position startPosition(boost::python::extract(pyStartPosition.attr("line")), + boost::python::extract(pyStartPosition.attr("column"))); + model::Position endPosition(boost::python::extract(pyEndPosition.attr("line")), + boost::python::extract(pyEndPosition.attr("column"))); + + fileLoc.range = model::Range(startPosition, endPosition); + + return fileLoc; +} + +typedef boost::shared_ptr PersistencePtr; + +BOOST_PYTHON_MODULE(persistence){ + boost::python::class_("Persistence", boost::python::init()) + .def("f", &Persistence::f) + .def("persist_file", &Persistence::persistFile) + .def("persist_variable", &Persistence::persistVariable) + .def("persist_function", &Persistence::persistFunction) + .def("persist_class", &Persistence::persistClass) + .def("persist_import", &Persistence::persistImport); +} + +PythonParser::PythonParser(ParserContext &ctx_) : AbstractParser(ctx_) {} + +PythonParser::~PythonParser() {} + +void PythonParser::markModifiedFiles() {} + +bool PythonParser::cleanupDatabase() { return true; } + +bool PythonParser::parse() +{ + setenv("PYTHONPATH", + "/home/rmfcnb/ELTE/Code-Compass-Python-Plugin/:" + "/home/rmfcnb/CodeCompass/build/plugins/python/parser/:" + "/usr/lib/python3.8/", 1); + Py_Initialize(); + init_module_persistence(); + + boost::python::object module = boost::python::import("my_ast.python_parser"); + + if(!module.is_none()){ + boost::python::object func = module.attr("parse"); + + if(!func.is_none() && PyCallable_Check(func.ptr())){ + std::string source_path; + for (const std::string& input : _ctx.options["input"].as>()){ + if (boost::filesystem::is_directory(input)){ + source_path = input; + } + } + if(source_path.empty()){ + std::cout << "No source path was found" << std::endl; + } else { + PersistencePtr persistencePtr(new Persistence(_ctx)); + + func(source_path, boost::python::ptr(persistencePtr.get())); + } + } else { + std::cout << "Cannot find function" << std::endl; + } + } else { + std::cout << "Cannot import module" << std::endl; + } + + // Py_Finalize(); + return true; +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ +boost::program_options::options_description getOptions() +{ + boost::program_options::options_description description("Python Plugin"); + description.add_options() + ("skip-doccomment", + "If this flag is given the parser will skip parsing the documentation " + "comments."); + return description; +} + +std::shared_ptr make(ParserContext& ctx_) +{ + return std::make_shared(ctx_); +} +} +#pragma clang diagnostic pop + +} +} \ No newline at end of file diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt new file mode 100644 index 000000000..aa75f0485 --- /dev/null +++ b/plugins/python/service/CMakeLists.txt @@ -0,0 +1,28 @@ +include_directories( + include + ${PROJECT_SOURCE_DIR}/util/include + ${PROJECT_SOURCE_DIR}/model/include + ${PROJECT_BINARY_DIR}/service/language/gen-cpp + ${PROJECT_BINARY_DIR}/service/project/gen-cpp + ${PROJECT_SOURCE_DIR}/service/project/include + ${PLUGIN_DIR}/model/include) + +include_directories(SYSTEM + ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) + +add_library(pythonservice SHARED + src/pythonservice.cpp) + +target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) + +target_link_libraries(pythonservice + util + model + cppmodel + mongoose + projectservice + languagethrift + gvc + ${THRIFT_LIBTHRIFT_LIBRARIES}) + +install(TARGETS pythonservice DESTINATION ${INSTALL_SERVICE_DIR}) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h new file mode 100644 index 000000000..48a5753fe --- /dev/null +++ b/plugins/python/service/include/service/pythonservice.h @@ -0,0 +1,239 @@ +#ifndef CC_SERVICE_LANGUAGE_PYTHONSERVICE_H +#define CC_SERVICE_LANGUAGE_PYTHONSERVICE_H + +#include + +#include + +#include +#include + +#include +#include + +namespace cc +{ +namespace service +{ +namespace language +{ + +class PythonServiceHandler : virtual public LanguageServiceIf +{ +public: + PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_); + + void getFileTypes(std::vector& return_) override; + + void getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) override; + + void getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) override; + + void getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) override; + + void getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) override; + + void getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) override; + + void getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) override; + + void getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) override; + + void getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) override; + + void getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& tags_) override; + + std::int32_t getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) override; + + void getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) override; + + void getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) override; + + void getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) override; + + void getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + std::int32_t getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) override; + + void getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) override; + +private: + enum ReferenceType + { + DECLARATION, /*!< By this options the declaration(s) of the AST node can be + queried. */ + + USAGE, /*!< By this option the usages of the AST node can be queried, i.e. + the nodes of which the mangled name is identical to the queried one. The + parser creates a mangled name even for variables and types, which is + unique for the element. */ + + THIS_CALLS, /*!< Get function calls in a function. WARNING: If the + definition of the AST node is not unique then it returns the callees of + one of them. */ + + CALLS_OF_THIS, /*!< Get calls of a function. */ + + CALLEE, /*!< Get called functions definitions. WARNING: If the definition of + the AST node is not unique then it returns the callees of one of them. */ + + CALLER, /*!< Get caller functions. */ + +// VIRTUAL_CALL, /*!< A function may be used virtually on a base type object. +// The exact type of the object is based on dynamic information, which can't +// be determined statically. Weak usage returns these possible calls. */ + +// FUNC_PTR_CALL, /*!< Functions can be assigned to function pointers which +// can be invoked later. This option returns these invocations. */ + + PARAMETER, /*!< This option returns the parameters of a function. */ + + LOCAL_VAR, /*!< This option returns the local variables of a function. */ + + RETURN_TYPE, /*!< This option returns the return type of a function. */ + +// OVERRIDE, /*!< This option returns the functions which the given function +// overrides. */ +// +// OVERRIDDEN_BY, /*!< This option returns the overrides of a function. */ +// +// READ, /*!< This option returns the places where a variable is read. */ +// +// WRITE, /*!< This option returns the places where a variable is written. */ + + TYPE, /*!< This option returns the type of a variable. */ + + INHERIT_FROM, /*!< Types from which the queried type inherits. */ + + INHERIT_BY, /*!< Types by which the queried type is inherited. */ + + DATA_MEMBER, /*!< Data members of a class. */ + + METHOD, /*!< Members of a class. */ + + NESTED_CLASS /*!< Nested classes. */ + }; + + enum FileReferenceType + { + IMPORTS, /*!< Imported modules in the current module. */ + + CLASSES, /*!< User defined classes */ + + FUNCTIONS, /*!< Functions in the current module. */ + + VARIABLES /*!< Variables in the current module. */ + }; + + static bool compareByPosition( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs); + + static bool compareByValue( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs); + + model::PythonAstNode queryPythonAstNode(const core::AstNodeId& astNodeId_); + + std::vector queryPythonAstNodes( + const core::AstNodeId& astNodeId_, + const odb::query& query_ + = odb::query(true)); + + std::vector queryPythonAstNodesInFile( + const core::FileId& fileId_, + const odb::query& query_ + = odb::query(true)); + + std::uint32_t queryPythonAstNodeCountInFile( + const core::FileId& fileId_, + const odb::query& query_ + = odb::query(true)); + + std::vector queryDeclarations(const core::AstNodeId& astNodeId_) + + odb::query astCallsQuery(const model::PythonAstNode& astNode_); + + std::vector queryCalls(const core::AstNodeId& astNodeId_); + + std::size_t queryPythonAstNodeCount( + const core::AstNodeId& astNodeId_, + const odb::query& query_ + = odb::query(true)); + + std::size_t queryCallsCount(const core::AstNodeId& astNodeId_); + + std::shared_ptr _db; + util::OdbTransaction _transaction; + + std::shared_ptr _datadir; + const cc::webserver::ServerContext& _context; +}; + +} +} +} + +#endif diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp new file mode 100644 index 000000000..9e9488b5d --- /dev/null +++ b/plugins/python/service/src/pythonservice.cpp @@ -0,0 +1,978 @@ +#include + +#include + +#include +//#include +#include +//#include +#include +//#include +#include +//#include +#include +//#include +#include +//#include +#include +//#include +#include +//#include + +namespace +{ + typedef odb::query AstQuery; + typedef odb::result AstResult; + typedef odb::query VarQuery; + typedef odb::result VarResult; + typedef odb::query FuncQuery; + typedef odb::result FuncResult; + typedef odb::query ClassQuery; + typedef odb::result ClassResult; + typedef odb::query ClassMemQuery; + typedef odb::result ClassMemResult; + typedef odb::query ModImpQuery; + typedef odb::result ModImpResult; + typedef odb::query InhQuery; + typedef odb::result InhResult; + + cc::service::language::AstNodeInfo createAstNodeInfo(const cc::model::PythonAstNode& astNode_) + { + cc::service::language::AstNodeInfo ret; + + ret.__set_id(std::to_string(astNode_.id)); + ret.__set_mangledNameHash(0); + ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); + ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); + ret.__set_astNodeValue(astNode_.astValue); + + ret.range.range.startpos.line = astNode_.location.range.start.line; + ret.range.range.startpos.column = astNode_.location.range.start.column; + ret.range.range.endpos.line = astNode_.location.range.end.line; + ret.range.range.endpos.column = astNode_.location.range.end.column; + + if (astNode_.location.file){ + ret.range.file = std::to_string(astNode_.location.file.object_id()); + } + + return ret; + } +} + +namespace cc { +namespace service { +namespace language { + +PythonServiceHandler::PythonServiceHandler( + std::shared_ptr db_, + std::shared_ptr datadir_, + const cc::webserver::ServerContext& context_) + : _db(db), + _transaction(db_), + _datadir(datadir_), + _context(context_) +{ +} + +void PythonServiceHandler::getFileTypes(std::vector& return_) +{ + return_.push_back("PY"); + return_.push_back("Dir"); +} + +void PythonServiceHandler::getAstNodeInfo( + AstNodeInfo& return_, + const core::AstNodeId& astNodeId_) +{ + return_ = _transaction([this, &astNodeId_](){ + return createAstNodeInfo(queryPythonAstNode(astNodeId_)); + }); +} + +void PythonServiceHandler::getAstNodeInfoByPosition( + AstNodeInfo& return_, + const core::FilePosition& fpos_) +{ + _transaction([&, this](){ + //--- Query nodes at the given position ---// + + AstResult nodes = _db->query( + AstQuery::location.file == std::stoull(fpos_.file) && + // StartPos <= Pos + ((AstQuery::location.range.start.line == fpos_.pos.line && + AstQuery::location.range.start.column <= fpos_.pos.column) || + AstQuery::location.range.start.line < fpos_.pos.line) && + // Pos < EndPos + ((AstQuery::location.range.end.line == fpos_.pos.line && + AstQuery::location.range.end.column > fpos_.pos.column) || + AstQuery::location.range.end.line > fpos_.pos.line)); + + //--- Select innermost clickable node ---// + + model::Range minRange(model::Position(0, 0), model::Position()); + model::PythonAstNode min; + + for (const model::PythonAstNode& node : nodes) + { + if (node.location.range < minRange) + { + min = node; + minRange = node.location.range; + } + } + + return_ = _transaction([this, &min](){ + return createAstNodeInfo(min); + }); + }); +} + +void PythonServiceHandler::getSourceText( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + return_ = _transaction([this, &astNodeId_](){ + model::PythonAstNode astNode = queryPythonAstNode(astNodeId_); + + if (astNode.location.file){ + return cc::util::textRange( + astNode.location.file.load()->content.load()->content, + astNode.location.range.start.line, + astNode.location.range.start.column, + astNode.location.range.end.line, + astNode.location.range.end.column); + } + + return std::string(); + }); +} + +void PythonServiceHandler::getDocumentation( + std::string& return_, + const core::AstNodeId& astNodeId_) +{ + // The ast module does not include comments (~documentation). + // TODO: try to use the tokenize module. + _return = std::string(); +} + +void PythonServiceHandler::getProperties( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + _transaction([&, this](){ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + switch (node.symbolType) + { + case model::PythonAstNode::SymbolType::Variable: + { + VarResult variables = _db->query( + VarQuery::astNodeId == node.id); + model::PythonVariable variable = *variables.begin(); + + return_["Name"] = variable.name; + return_["Qualified name"] = variable.qualifiedName; + return_["Type"] = variable.qualifiedType; + break; + } + + case model::PythonAstNode::SymbolType::Function: + { + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + return_["Name"] = function.qualifiedName.substr( + function.qualifiedName.find_last_of(':') + 1); + return_["Qualified name"] = function.qualifiedName; + return_["Signature"] = function.name; + + break; + } + + case model::PythonAstNode::SymbolType::Class: + { + ClassResult classes = _db->query( + ClassQuery::astNodeId == node.id); + model::PythonCLass cl = *classes.begin(); + + return_["Name"] = cl.name; + return_["Qualified name"] = cl.qualifiedName; + + break; + } + + case model::PythonAstNode::SymbolType::Module: + { + ModImpResult modules = _db->query( + ModImpQuery::astNodeId == node.id); + model::PythonModule module = *modules.begin(); + + return_["Name"] = module.name; + return_["Qualified name"] = module.qualifiedName; + + break; + } + } + }); +} + +void PythonServiceHandler::getDiagramTypes( + std::map& return_, + const core::AstNodeId& astNodeId_) +{ + +} + +void PythonServiceHandler::getDiagram( + std::string& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getFileDiagramTypes( + std::map& return_, + const core::FileId& fileId_) +{ + +} + +void PythonServiceHandler::getFileDiagram( + std::string& return_, + const core::FileId& fileId_, + const int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getFileDiagramLegend( + std::string& return_, + const std::int32_t diagramId_) +{ + +} + +void PythonServiceHandler::getReferenceTypes( + std::map& return_, + const core::AstNodeId& astNodeId) +{ + model::PythonAstNode node = queryCppAstNode(astNodeId_); + + return_["Declaration"] = DECLARATION; + return_["Usage"] = USAGE; + + switch (node.symbolType){ + case model::PythonAstNode::SymbolType::Variable: + return_["Type"] = TYPE; + break; + + case model::PythonAstNode::SymbolType::Function: + return_["This calls"] = THIS_CALLS; + return_["Callee"] = CALLEE; + return_["Caller"] = CALLER; + return_["Parameters"] = PARAMETER; + return_["Local variables"] = LOCAL_VAR; + return_["Return type"] = RETURN_TYPE; + break; + + case model::PythonAstNode::SymbolType::Class: + return_["Inherits from"] = INHERIT_FROM; + return_["Inherited by"] = INHERIT_BY; + return_["Data member"] = DATA_MEMBER; + return_["Method"] = METHOD; + return_["Nested class"] = NESTED_CLASS; + break; + + case model::PythonAstNode::SymbolType::Module: + break; + } +} + +void PythonServiceHandler::getReferences( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::vector& /*tags_*/) +{ + std::vector nodes; + model::PythonAstNode node; + + return _transaction([&, this](){ + switch (referenceId_) + { + case DECLARATION: + nodes = queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + break; + + case USAGE: + nodes = queryPythonAstNodes(astNodeId_); + break; + + case THIS_CALLS: + nodes = queryCalls(astNodeId_); + break; + + case CALLS_OF_THIS: + nodes = queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Usage); + break; + + case CALLEE: + for (const model::PythonAstNode& call : queryCalls(astNodeId_)) + { + core::AstNodeId astNodeId = std::to_string(call.id); + std::vector defs = queryDefinitions(astNodeId); + nodes.insert(nodes.end(), defs.begin(), defs.end()); + } + + std::sort(nodes.begin(), nodes.end()); + nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); + + break; + + case CALLER: + for (const model::PythonAstNode& astNode : queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::CppAstNode::AstType::Usage)) + { + const model::Position& start = astNode.location.range.start; + const model::Position& end = astNode.location.range.end; + + AstResult result = _db->query( + AstQuery::astType == model::PythonAstNode::AstType::Declaration && + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + // Same file + AstQuery::location.file == astNode.location.file.object_id() && + // StartPos >= Pos + ((AstQuery::location.range.start.line == start.line && + AstQuery::location.range.start.column <= start.column) || + AstQuery::location.range.start.line < start.line) && + // Pos > EndPos + ((AstQuery::location.range.end.line == end.line && + AstQuery::location.range.end.column > end.column) || + AstQuery::location.range.end.line > end.line)); + + nodes.insert(nodes.end(), result.begin(), result.end()); + } + + std::sort(nodes.begin(), nodes.end()); + nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); + + break; + + case PARAMETER: + { + node = queryPythonAstNode(astNodeId_); + + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + for (auto var : function.parameters){ + nodes.push_back(queryPythonAstNode( + std::to_string(var.load()->astNodeId))); + } + + break; + } + + case LOCAL_VAR: + { + node = queryPythonAstNode(astNodeId_); + + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + for (auto var : function.locals){ + nodes.push_back(queryPythonAstNode( + std::to_string(var.load()->astNodeId))); + } + + break; + } + + case RETURN_TYPE: + { + node = queryPythonAstNode(astNodeId_); + + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + model::PythonFunction function = *functions.begin(); + + ClassResult result = _db->query( + ClassQuery::astNodeId == function.typeId); + + for (const model::PythonClass& cl : result) + { + std::vector defs = + queryDeclarations(std::to_string(cl.astNodeId)); + nodes.insert(nodes.end(), defs.begin(), defs.end()); + } + + break; + } + + case TYPE: + { + node = queryPythonAstNode(astNodeId_); + + VarResult varNodes = _db->query( + VarQuery::astNodeId == node.id); + + const model::PythonVariable& variable = *varNodes.begin(); + + ClassResult result = _db->query( + ClassQuery::astNodeId == variable.typeId); + + for (const model::PythonClass& cl : result) + { + std::vector defs = + queryDeclarations(std::to_string(cl.astNodeId)); + nodes.insert(nodes.end(), defs.begin(), defs.end()); + } + + break; + } + + case INHERIT_FROM: + node = queryCppAstNode(astNodeId_); + + for (const model::PythonInheritance& inh : + _db->query( + InhQuery::derived == node.id)) + { + AstResult result = _db->query( + AstQuery::astNodeId == inh.base && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + nodes.insert(nodes.end(), result.begin(), result.end()); + } + + break; + + case INHERIT_BY: + node = queryCppAstNode(astNodeId_); + + for (const model::PythonInheritance& inh : + _db->query( + InhQuery::base == node.id )) + { + AstResult result = _db->query( + AstQuery::astNodeId == inh.derived && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); + nodes.insert(nodes.end(), result.begin(), result.end()); + } + + break; + + case DATA_MEMBER: + { + node = queryPythonAstNode(astNodeId_); + + for (const model::PythonClassMember& mem : _db->query( + ClassMemQuery::astNodeId == node.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Field)) + { + for (const model::PythonVariable& var : _db->query( + VarQuery::id == mem.memberId)) + { + model::PythonAstNode astNode = queryPythonAstNode(var.astNodeId); + if (astNode.location.range.end.line != model::Position::npos){ + nodes.push_back(astNode); + } + } + } + + break; + } + + case METHOD: + { + node = queryPythonAstNode(astNodeId_); + + for (const model::PythonClassMember& mem : _db->query( + ClassMemQuery::astNodeId == node.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Method)) + { + for (const model::PythonFunction& var : _db->query( + FuncQuery::id == mem.memberId)) + { + nodes.push_back(queryPythonAstNode(var.astNodeId)); + } + } + + break; + } + + case NESTED_CLASS: + { + node = queryPythonAstNode(astNodeId_); + + for (const model::PythonClassMember& mem : _db->query( + ClassMemQuery::astNodeId == node.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Class)) + { + for (const model::PythonClass& cl : _db->query( + ClassQuery::id == mem.memberId)) + { + nodes.push_back(queryPythonAstNode(cl.astNodeId)); + } + } + + break; + } + } + + std::sort(nodes.begin(), nodes.end(), compareByValue); + + return_.reserve(nodes.size()); + _transaction([this, &return_, &nodes](){ + std::transform( + nodes.begin(), nodes.end(), + std::back_inserter(return_), + createAstNodeInfo); + }); + }); +} + +std::int32_t PythonServiceHandler::getReferenceCount( + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + return _transaction([&, this]() -> std::int32_t { + switch (referenceId_) + { + case DECLARATION: + return queryPythonAstNodeCount(astNodeId_, + AstQuery::astType == model::CppAstNode::AstType::Declaration); + + case USAGE: + return queryPythonAstNodeCount(astNodeId_); + + case THIS_CALLS: + return queryCallsCount(astNodeId_); + + case CALLS_OF_THIS: + return queryPythonAstNodeCount(astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Usage); + + case CALLEE: + { + std::int32_t count = 0; + + std::set astNodeIds; + for (const model::PythonAstNode& call : queryCalls(astNodeId_)) + { + model::PythonAstNode node = queryPythonAstNode(std::to_string(call.id)); + astNodeIds.insert(node.id); + } + + if (!astNodeIds.empty()){ + count = _db->query_value( + AstQuery::id.in_range( + astNodeIds.begin(), astNodeIds.end()) && + AstQuery::astType == model::PythonAstNode::AstType::Declaration && + AstQuery::location.range.end.line != model::Position::npos).count; + } + + return count; + } + + case CALLER: + { + std::vector references; + getReferences(references, astNodeId_, CALLER, {}); + return references.size(); + } + + case PARAMETER: + return _db->query_value( + FuncQuery::astNodeId == node.id).count; + + case LOCAL_VAR: + return _db->query_value( + FuncQuery::astNodeId == node.id).count; + + case RETURN_TYPE: + { + node = queryPythonAstNode(astNodeId_); + + FuncResult functions = _db->query( + FuncQuery::astNodeId == node.id); + + const model::PythonFunction& function = *functions.begin(); + + return _db->query_value( + ClassQuery::id == function.typeId).count; + + break; + } + + case TYPE: + { + node = queryPythonAstNode(astNodeId_); + + VarResult varNodes = _db->query( + VarQuery::astNodeId == node.id); + + const model::PythonVariable& variable = *varNodes.begin(); + + return _db->query_value( + ClassQuery::id == variable.typeId).count; + } + + case INHERIT_FROM: + return _db->query_value( + InhQuery::derived == node.id).count; + + case INHERIT_BY: + return _db->query_value( + InhQuery::base == node.id).count; + + case DATA_MEMBER: + return _db->query_value( + ClassMemQuery::astNodeId == node.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Field).count; + + case METHOD: + return _db->query_value( + ClassMemQuery::astNodeId == node.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Method).count; + + case NESTED_CLASS: + return _db->query_value( + ClassMemQuery::astNodeId == node.id && + ClassMemQuery::kind == model::PythonClassMember::Kind::Class).count; + + default: + return 0; + } + }); +} + +void PythonServiceHandler::getReferencesInFile( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const core::FileId& fileId_, + const std::vector& tags_) +{ + // TODO +} + +void PythonServiceHandler::getReferencesPage( + std::vector& return_, + const core::AstNodeId& astNodeId_, + const std::int32_t referenceId_, + const std::int32_t pageSize_, + const std::int32_t pageNo_) +{ + // TODO +} + +void PythonServiceHandler::getFileReferenceTypes( + std::map& return_, + const core::FileId& fileId_) +{ + return_["Imports"] = IMPORTS; + return_["Classes"] = CLASSES; + return_["Functions"] = FUNCTIONS; + return_["Variables"] = VARIABLES; +} + +void PythonServiceHandler::getFileReferences( + std::vector& return_, + const core::FileId& fileId_, + const std::int32_t referenceId_) +{ + std::vector nodes; + + _transaction([&, this](){ + switch (referenceId_) + { + case CLASSES: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Class); + break; + + case VARIABLES: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Variable && + AstQuery::astType == model::CppAstNode::AstType::Declaration); + break; + + case FUNCTIONS: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Function && + (AstQuery::astType == model::CppAstNode::AstType::Declaration)); + break; + + case IMPORTS: + nodes = queryPythonAstNodesInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Import); + break; + } + + std::sort(nodes.begin(), nodes.end(), compareByValue); + + return_.reserve(nodes.size()); + _transaction([this, &return_, &nodes](){ + std::transform( + nodes.begin(), nodes.end(), + std::back_inserter(return_), + createAstNodeInfo); + }); + }); +} + +std::int32_t PythonServiceHandler::getFileReferenceCount( + const core::FileId& fileId_, + const std::int32_t referenceId_) +{ + return _transaction([&, this]() -> std::int32_t { + switch (referenceId_) + { + case CLASSES: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Class); + break; + + case VARIABLES: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Variable && + AstQuery::astType == model::CppAstNode::AstType::Declaration); + break; + + case FUNCTIONS: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Function && + AstQuery::astType == model::CppAstNode::AstType::Declaration)); + break; + + case IMPORTS: + return queryPythonAstNodeCountInFile(fileId_, + AstQuery::symbolType == model::CppAstNode::SymbolType::Import); + break; + + default: + return 0; + } + }); +} + +void PythonServiceHandler::getSyntaxHighlight( + std::vector& return_, + const core::FileRange& range_) +{ + std::vector content; + + _transaction([&, this]() { + + //--- Load the file content and break it into lines ---// + + model::FilePtr file = _db->query_one( + FileQuery::id == std::stoull(range_.file)); + + if (!file->content.load()){ + return; + } + + std::istringstream s(file->content->content); + std::string line; + while (std::getline(s, line)){ + content.push_back(line); + } + + //--- Iterate over AST node elements ---// + + for (const model::PythonAstNode& node : _db->query( + AstQuery::location.file == std::stoull(range_.file) && + AstQuery::location.range.start.line >= range_.range.startpos.line && + AstQuery::location.range.end.line < range_.range.endpos.line && + AstQuery::location.range.end.line != model::Position::npos)) + { + if (node.astValue.empty()){ + continue; + } + + // Regular expression to find element position + std::string reg = "\\b" + node.astValue + "\\b"; + + for (std::size_t i = node.location.range.start.line - 1; + i < node.location.range.end.line && i < content.size(); + ++i) + { + std::regex words_regex(reg); + auto words_begin = std::sregex_iterator( + content[i].begin(), content[i].end(), + words_regex); + auto words_end = std::sregex_iterator(); + + for (std::sregex_iterator ri = words_begin; ri != words_end; ++ri) + { + SyntaxHighlight syntax; + syntax.range.startpos.line = i + 1; + syntax.range.startpos.column = ri->position() + 1; + syntax.range.endpos.line = i + 1; + syntax.range.endpos.column = + syntax.range.startpos.column + node.astValue.length(); + + std::string symbolClass = + "cm-" + model::symbolTypeToString(node.symbolType); + syntax.className = symbolClass + " " + + symbolClass + "-" + model::astTypeToString(node.astType); + + return_.push_back(std::move(syntax)); + } + } + } + }); +} + +bool PythonServiceHandler::compareByPosition( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs) +{ + return lhs.location.range.start < rhs.location.range.start; +} + +bool PythonServiceHandler::compareByValue( + const model::PythonAstNode& lhs, + const model::PythonAstNode& rhs) +{ + return lhs.astValue < rhs.astValue; +} + +model::PythonAstNode PythonServiceHandler::queryPythonAstNode(const core::AstNodeId& astNodeId_) +{ + return _transaction([&, this](){ + model::PythonAstNode node; + + if (!_db->find(std::stoull(astNodeId_), node)) + { + core::InvalidId ex; + ex.__set_msg("Invalid PythonAstNode ID"); + ex.__set_nodeid(astNodeId_); + throw ex; + } + + return node; + }); +} + +std::vector PythonServiceHandler::queryPythonAstNodes( + const core::AstNodeId& astNodeId_, + const odb::query& query_) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + AstResult result = _db->query( + AstQuery::astNodeId == node.id && + AstQuery::location.range.end.line != model::Position::npos && + query_); + + return std::vector(result.begin(), result.end()); +} + +std::vector PythonServiceHandler::queryPythonAstNodesInFile( + const core::FileId& fileId_, + const odb::query& query_) +{ + AstResult result = _db->query( + AstQuery::location.file == std::stoull(fileId_) && query_); + + return std::vector(result.begin(), result.end()); +} + +std::uint32_t PythonServiceHandler::queryPythonAstNodeCountInFile( + const core::FileId& fileId_, + const odb::query& query_) +{ + return _db->query_value( + AstQuery::location.file == std::stoull(fileId_) && query_).count; +} + +std::vector PythonServiceHandler::queryDeclarations(const core::AstNodeId& astNodeId_) +{ + return queryPythonAstNodes( + astNodeId_, + AstQuery::astType == model::PythonAstNode::AstType::Declaration); +} + +odb::query PythonServiceHandler::astCallsQuery(const model::PythonAstNode& astNode_) +{ + const model::Position& start = astNode_.location.range.start; + const model::Position& end = astNode_.location.range.end; + + return (AstQuery::location.file == astNode_.location.file.object_id() && + (AstQuery::astType == model::PythonAstNode::AstType::Usage || + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + // StartPos >= Pos + ((AstQuery::location.range.start.line == start.line && + AstQuery::location.range.start.column >= start.column) || + AstQuery::location.range.start.line > start.line) && + // Pos > EndPos + ((AstQuery::location.range.end.line == end.line && + AstQuery::location.range.end.column < end.column) || + AstQuery::location.range.end.line < end.line)); +} + +std::vector PythonServiceHandler::queryCalls(const core::AstNodeId& astNodeId_) +{ + std::vector nodes = queryDeclarations(astNodeId_); + + if (nodes.empty()){ + return nodes; + } + + model::PythonAstNode node = nodes.front(); + AstResult result = _db->query(astCallsQuery(node)); + + return std::vector(result.begin(), result.end()); +} + +std::size_t PythonServiceHandler::queryPythonAstNodeCount( + const core::AstNodeId& astNodeId_, + const odb::query& query_) +{ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + + model::PythonAstCount q = _db->query_value( + AstQuery::astNodeId == node.id && + AstQuery::location.range.end.line != model::Position::npos && + query_); + + return q.count; +} + +std::size_t queryCallsCount(const core::AstNodeId& astNodeId_) +{ + std::vector nodes = queryDeclarations(astNodeId_); + + if (nodes.empty()){ + return std::size_t(0); + } + + model::PythonAstNode node = nodes.front(); + + return _db->query_value(astCallsQuery(node)).count; +} + +} +} +} \ No newline at end of file diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js new file mode 100644 index 000000000..f2eb1c797 --- /dev/null +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -0,0 +1,14 @@ +require([ + 'codecompass/model', + 'codecompass/viewHandler', + 'codecompass/util'], +function (model, viewHandler, util){ + model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + var pythonInfoTree = {}; + + viewHandler.registerModule(pythonInfoTree, { + type : viewHandler.moduleType.InfoTree, + service : model.pythonservice + }); +}); \ No newline at end of file diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js new file mode 100644 index 000000000..ebf31c5fc --- /dev/null +++ b/plugins/python/webgui/js/pythonMenu.js @@ -0,0 +1,11 @@ +require([ + 'dojo/topic', + 'dijit/Menu', + 'dijit/MenuItem', + 'dijit/PopupMenuItem', + 'codecompass/astHelper', + 'codecompass/model', + 'codecompass/viewHandler'], +function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, viewHandler){ + model.addService('pythonservice', 'PythonService', LanguageServiceClient); +}); \ No newline at end of file From 97f7ff33004ecdd28fe5613e15c09cb79acc82e4 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 1 Nov 2020 19:46:10 +0100 Subject: [PATCH 02/39] Fix python service and parser to compile --- plugins/python/parser/CMakeLists.txt | 7 +- plugins/python/parser/src/my_ast/__init__.py | 7 - plugins/python/parser/src/my_ast/base_data.py | 66 -- .../parser/src/my_ast/built_in_functions.py | 612 ------------ .../parser/src/my_ast/built_in_operators.py | 324 ------- .../parser/src/my_ast/built_in_types.py | 174 ---- .../python/parser/src/my_ast/class_data.py | 55 -- .../src/my_ast/class_init_declaration.py | 16 - .../parser/src/my_ast/class_preprocessor.py | 84 -- .../parser/src/my_ast/common/__init__.py | 1 - .../parser/src/my_ast/common/file_position.py | 20 - .../parser/src/my_ast/common/hashable_list.py | 31 - .../parser/src/my_ast/common/location.py | 8 - .../parser/src/my_ast/common/parser_tree.py | 30 - .../parser/src/my_ast/common/position.py | 36 - .../parser/src/my_ast/common/unique_list.py | 23 - .../python/parser/src/my_ast/common/utils.py | 18 - plugins/python/parser/src/my_ast/db_test.log | 27 - plugins/python/parser/src/my_ast/file_info.py | 59 -- .../python/parser/src/my_ast/function_data.py | 41 - .../src/my_ast/function_symbol_collector.py | 64 -- .../function_symbol_collector_factory.py | 14 - .../python/parser/src/my_ast/import_finder.py | 27 - .../parser/src/my_ast/import_preprocessor.py | 129 --- plugins/python/parser/src/my_ast/logger.py | 16 - plugins/python/parser/src/my_ast/main.py | 43 - .../src/my_ast/member_access_collector.py | 210 ----- .../parser/src/my_ast/parse_exception.py | 20 - plugins/python/parser/src/my_ast/parser.py | 127 --- .../parser/src/my_ast/persistence/__init__.py | 2 - .../parser/src/my_ast/persistence/base_dto.py | 18 - .../src/my_ast/persistence/build_action.py | 11 - .../my_ast/persistence/build_source_target.py | 14 - .../src/my_ast/persistence/class_dto.py | 23 - .../src/my_ast/persistence/documented_dto.py | 3 - .../parser/src/my_ast/persistence/file.py | 38 - .../src/my_ast/persistence/file_content.py | 3 - .../my_ast/persistence/file_content_dto.py | 3 - .../parser/src/my_ast/persistence/file_dto.py | 23 - .../src/my_ast/persistence/function_dto.py | 12 - .../src/my_ast/persistence/import_dto.py | 17 - .../src/my_ast/persistence/persistence.py | 48 - .../src/my_ast/persistence/variable_dto.py | 5 - .../placeholder_function_declaration_cache.py | 49 - .../parser/src/my_ast/preprocessed_data.py | 8 - .../parser/src/my_ast/preprocessed_file.py | 59 -- .../src/my_ast/preprocessed_function.py | 19 - .../src/my_ast/preprocessed_variable.py | 19 - .../python/parser/src/my_ast/python_parser.py | 30 - plugins/python/parser/src/my_ast/scope.py | 177 ---- .../python/parser/src/my_ast/scope_manager.py | 383 -------- .../parser/src/my_ast/symbol_collector.py | 881 ------------------ .../src/my_ast/symbol_collector_interface.py | 27 - .../python/parser/src/my_ast/symbol_finder.py | 10 - plugins/python/parser/src/my_ast/test.log | 27 - plugins/python/parser/src/my_ast/type_data.py | 52 -- .../parser/src/my_ast/type_deduction.py | 507 ---------- .../python/parser/src/my_ast/variable_data.py | 77 -- plugins/python/parser/src/pythonparser.cpp | 101 +- plugins/python/service/CMakeLists.txt | 2 +- .../service/include/service/pythonservice.h | 8 +- plugins/python/service/src/pythonservice.cpp | 183 ++-- 62 files changed, 193 insertions(+), 4935 deletions(-) delete mode 100644 plugins/python/parser/src/my_ast/__init__.py delete mode 100644 plugins/python/parser/src/my_ast/base_data.py delete mode 100644 plugins/python/parser/src/my_ast/built_in_functions.py delete mode 100644 plugins/python/parser/src/my_ast/built_in_operators.py delete mode 100644 plugins/python/parser/src/my_ast/built_in_types.py delete mode 100644 plugins/python/parser/src/my_ast/class_data.py delete mode 100644 plugins/python/parser/src/my_ast/class_init_declaration.py delete mode 100644 plugins/python/parser/src/my_ast/class_preprocessor.py delete mode 100644 plugins/python/parser/src/my_ast/common/__init__.py delete mode 100644 plugins/python/parser/src/my_ast/common/file_position.py delete mode 100644 plugins/python/parser/src/my_ast/common/hashable_list.py delete mode 100644 plugins/python/parser/src/my_ast/common/location.py delete mode 100644 plugins/python/parser/src/my_ast/common/parser_tree.py delete mode 100644 plugins/python/parser/src/my_ast/common/position.py delete mode 100644 plugins/python/parser/src/my_ast/common/unique_list.py delete mode 100644 plugins/python/parser/src/my_ast/common/utils.py delete mode 100644 plugins/python/parser/src/my_ast/db_test.log delete mode 100644 plugins/python/parser/src/my_ast/file_info.py delete mode 100644 plugins/python/parser/src/my_ast/function_data.py delete mode 100644 plugins/python/parser/src/my_ast/function_symbol_collector.py delete mode 100644 plugins/python/parser/src/my_ast/function_symbol_collector_factory.py delete mode 100644 plugins/python/parser/src/my_ast/import_finder.py delete mode 100644 plugins/python/parser/src/my_ast/import_preprocessor.py delete mode 100644 plugins/python/parser/src/my_ast/logger.py delete mode 100644 plugins/python/parser/src/my_ast/main.py delete mode 100644 plugins/python/parser/src/my_ast/member_access_collector.py delete mode 100644 plugins/python/parser/src/my_ast/parse_exception.py delete mode 100644 plugins/python/parser/src/my_ast/parser.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/__init__.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/base_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/build_action.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/build_source_target.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/class_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/documented_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/file.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/file_content.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/file_content_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/file_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/function_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/import_dto.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/persistence.py delete mode 100644 plugins/python/parser/src/my_ast/persistence/variable_dto.py delete mode 100644 plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py delete mode 100644 plugins/python/parser/src/my_ast/preprocessed_data.py delete mode 100644 plugins/python/parser/src/my_ast/preprocessed_file.py delete mode 100644 plugins/python/parser/src/my_ast/preprocessed_function.py delete mode 100644 plugins/python/parser/src/my_ast/preprocessed_variable.py delete mode 100644 plugins/python/parser/src/my_ast/python_parser.py delete mode 100644 plugins/python/parser/src/my_ast/scope.py delete mode 100644 plugins/python/parser/src/my_ast/scope_manager.py delete mode 100644 plugins/python/parser/src/my_ast/symbol_collector.py delete mode 100644 plugins/python/parser/src/my_ast/symbol_collector_interface.py delete mode 100644 plugins/python/parser/src/my_ast/symbol_finder.py delete mode 100644 plugins/python/parser/src/my_ast/test.log delete mode 100644 plugins/python/parser/src/my_ast/type_data.py delete mode 100644 plugins/python/parser/src/my_ast/type_deduction.py delete mode 100644 plugins/python/parser/src/my_ast/variable_data.py diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index e3709df09..0e98dd3f0 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -15,7 +15,12 @@ add_library(pythonparser SHARED find_package (Python) -target_link_libraries(pythonparser model ${PYTHON_LIBRARIES} ${Boost_LIBRARIES}) +target_link_libraries(pythonparser + util + model + pythonmodel + ${PYTHON_LIBRARIES} + ${Boost_LIBRARIES}) target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) target_link_options(pythonparser PUBLIC -Xlinker -export-dynamic) diff --git a/plugins/python/parser/src/my_ast/__init__.py b/plugins/python/parser/src/my_ast/__init__.py deleted file mode 100644 index 7d7338820..000000000 --- a/plugins/python/parser/src/my_ast/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -__all__ = ["base_data", "built_in_functions", "built_in_operators", "built_in_types", "class_data", - "class_init_declaration", "class_preprocessor", "file_info", "function_data", "function_symbol_collector", - "function_symbol_collector_factory", "import_finder", "import_preprocessor", "logger", "main", - "member_access_collector", "parser", "parse_exception", "placeholder_function_declaration_cache", - "preprocessed_data", "preprocessed_file", "preprocessed_function", "preprocessed_variable", "python_parser", - "scope", "scope_manager", "symbol_collector", "symbol_collector_interface", "symbol_finder", "type_data", - "type_deduction", "variable_data"] diff --git a/plugins/python/parser/src/my_ast/base_data.py b/plugins/python/parser/src/my_ast/base_data.py deleted file mode 100644 index 89c85cb97..000000000 --- a/plugins/python/parser/src/my_ast/base_data.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import List, TypeVar, Generic, Set, Optional - -from my_ast.common.file_position import FilePosition -from my_ast.type_data import DeclarationType - - -class Usage: - def __init__(self, name: str, position: FilePosition): - self.name = name - self.file_position = position - - def __str__(self): - return self.name + ' (' + str(self.file_position) + ')' - - def __repr__(self): - return self.__str__() - - -class Declaration(DeclarationType): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, - declaration_type: Optional[Set[DeclarationType]] = None): - DeclarationType.__init__(self, name, qualified_name) - assert declaration_type is None or not any(isinstance(x, type) for x in declaration_type) - self.file_position: FilePosition = pos - if declaration_type is None: - self.type: Set[DeclarationType] = set() - else: - self.type: Set[DeclarationType] = declaration_type - self.usages: List[Usage] = [] - - def is_same_declaration(self, other: 'Declaration') -> bool: - return self.__eq__(other) and self.file_position == other.file_position - - def is_same_usage(self, other: Usage): - return self.name == other.name and self.file_position == other.file_position - - def __eq__(self, other): - return isinstance(other, type(self)) and self.name == other.name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.name) - - def get_type_repr(self) -> str: - return '[' + ','.join({x.get_type_repr() if x is not None else 'None' for x in self.type}) + ']' - - -class TypeDeclaration(Declaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition): - super().__init__(name, qualified_name, pos) - self.type.add(self) - - -T = TypeVar('T', bound=Declaration) - - -class ImportedDeclaration(Generic[T]): - def __init__(self, declaration: T): - self.imported_declaration = declaration - - -class DocumentedType: - def __init__(self, documentation: str): - self.documentation: str = documentation diff --git a/plugins/python/parser/src/my_ast/built_in_functions.py b/plugins/python/parser/src/my_ast/built_in_functions.py deleted file mode 100644 index 66c46e358..000000000 --- a/plugins/python/parser/src/my_ast/built_in_functions.py +++ /dev/null @@ -1,612 +0,0 @@ -import inspect -import sys -from abc import ABC, abstractmethod -from typing import Optional, Set, Union, List - -from my_ast import built_in_types -from my_ast.built_in_types import Boolean, Complex, Dictionary, Float, FrozenSet, Integer, File, Range, Slice, Tuple, \ - Object, MemoryView, String, Bytes, ByteArray, Type -from my_ast.common.file_position import FilePosition -from my_ast.function_data import FunctionDeclaration -from my_ast.type_data import DeclarationType - - -class BuiltInFunction(FunctionDeclaration, ABC): - def __init__(self): - file_position = FilePosition.get_empty_file_position() - qualified_name = "builtins." + self.get_name() - super().__init__(self.get_name(), qualified_name, file_position, [], "", self.get_type()) - if self.get_override() is None: - self.override = [] - else: - self.override: List[str] = self.get_override() - - @staticmethod - @abstractmethod - def get_name() -> str: - pass - - @staticmethod - def get_override() -> Optional[List[str]]: - return None - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return None - - def get_type_repr(self) -> str: - if self.get_type() is None: - return "" - return ','.join([x.get_type_repr() for x in self.get_type()]) - - -class FunctionDecorator(BuiltInFunction, ABC): - pass - - -class AbsFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'abs' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__abs__'] - - -class AllFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'all' - - -class AnyFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'any' - - -class AsciiFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'ascii' - - -# returns binary string -class BinFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'bin' - - -class BoolFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'bool' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Boolean()} - - -class BreakpointFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'breakpoint' - - -class BytearrayFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'bytearray' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {ByteArray()} - - -class BytesFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'bytes' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Bytes()} - - -class CallableFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'callable' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Boolean()} - - -class ChrFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'chr' - - -class ClassmethodFunction(FunctionDecorator): - @staticmethod - def get_name() -> str: - return 'classmethod' - - -class CompileFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'compile' - - -class ComplexFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'complex' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__complex__', '__float__', '__index__'] - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Complex()} - - -class DelattrFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'delattr' - - -class DictFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'dict' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Dictionary()} - - -class DirFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'dir' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__dir__'] - - -class DivmodFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'divmod' - - -# returns enumerate object -class EnumerateFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'enumerate' - - -class EvalFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'eval' - - -class ExecFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'exec' - - -class FilterFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'filter' - - -class FloatFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'float' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Float()} - - -class FormatFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'format' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__format__'] - - -class FrozensetFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'frozenset' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {FrozenSet()} - - -class GetattrFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'getattr' - - -class GlobalsFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'globals' - - -class HasattrFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'hasattr' - - -class HashFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'hash' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__hash__'] - - -class HelpFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'help' - - -class HexFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'hex' - - -class IdFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'id' - - -class InputFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'input' - - -class IntFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'int' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__init__', '__index__', '__truncate__'] - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Integer()} - - -class IsinstanceFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'isinstance' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Boolean()} - - -class IssubclassFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'issubclass' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Boolean()} - - -# returns Iterator -class IterFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'iter' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__iter__'] - - -class LenFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'len' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Integer()} - - -class ListFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'list' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {built_in_types.List()} - - -class LocalsFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'locals' - - -class MapFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'map' - - -class MaxFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'max' - - -class MemoryViewFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'memoryview' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {MemoryView()} - - -class MinFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'min' - - -class NextFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'next' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__next__'] - - -class ObjectFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'object' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Object()} - - -class OctFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'oct' - - -class OpenFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'open' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {File()} - - -class OrdFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'ord' - - -class PowFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'pow' - - -class PrintFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'print' - - -class PropertyFunction(FunctionDecorator): - @staticmethod - def get_name() -> str: - return 'property' - - -class RangeFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'range' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Range()} - - -class ReprFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'repr' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__repr__'] - - -# returns iterator -class ReversedFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'reversed' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__reversed__'] - - -class RoundFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'round' - - -class SetFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'set' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {built_in_types.Set()} - - -class SetattrFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'setattr' - - -class SliceFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'slice' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Slice()} - - -class SortedFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'sorted' - - -class StaticmethodFunction(FunctionDecorator): - @staticmethod - def get_name() -> str: - return 'staticmethod' - - -class StrFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'str' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__str__', '__repr__'] - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {String()} - - -class SumFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'sum' - - -class SuperFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'super' - - -class TupleFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'tuple' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Tuple()} - - -class TypeFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'type' - - @staticmethod - def get_type() -> Optional[Set[Union[DeclarationType]]]: - return {Type()} - - -class VarsFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'vars' - - @staticmethod - def get_override() -> Optional[List[str]]: - return ['__dict__'] - - -class ZipFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return 'zip' - - -class ImportFunction(BuiltInFunction): - @staticmethod - def get_name() -> str: - return '__import__' - - -built_in_functions = inspect.getmembers(sys.modules[__name__], - lambda x: inspect.isclass(x) and issubclass(x, BuiltInFunction) and - x is not BuiltInFunction and x is not FunctionDecorator) - - -def get_built_in_function(name: str) -> Optional[BuiltInFunction]: - bif = [x for x in built_in_functions if x[1].get_name() == name] - assert len(bif) <= 1 - if len(bif) == 0: - return None - - return bif[0][1]() diff --git a/plugins/python/parser/src/my_ast/built_in_operators.py b/plugins/python/parser/src/my_ast/built_in_operators.py deleted file mode 100644 index 6383a0fb4..000000000 --- a/plugins/python/parser/src/my_ast/built_in_operators.py +++ /dev/null @@ -1,324 +0,0 @@ -import ast -import inspect -import sys -from abc import ABC, abstractmethod -from typing import Type, List, Optional - -from my_ast.built_in_types import Complex, Float, Integer, Boolean - - -class BuiltInOperator(ABC): - def __init__(self): - self.ast_type = self.get_ast_type() - self.override = self.get_override() - - @staticmethod - @abstractmethod - def get_ast_type() -> Type[ast.AST]: - pass - - @staticmethod - def get_override() -> List[str]: - return [] - - # @staticmethod - # @abstractmethod - # def get_type() -> Type: - # pass - - -class UnaryOperator(BuiltInOperator, ABC): - pass - - -class BinaryOperator(BuiltInOperator, ABC): - pass - - -class CompareOperator(BuiltInOperator, ABC): - pass - - -class BooleanOperator(BuiltInOperator, ABC): - pass - - -class AddOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Add - - @staticmethod - def get_override() -> List[str]: - return ['__add__'] - - -class SubOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Sub - - @staticmethod - def get_override() -> List[str]: - return ['__sub__'] - - -class MultOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Mult - - @staticmethod - def get_override() -> List[str]: - return ['__mul__'] - - -class MatMultOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.MatMult - - @staticmethod - def get_override() -> List[str]: - return ['__matmul__'] - - -class DivOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Div - - @staticmethod - def get_override() -> List[str]: - return ['__truediv__'] - - -class ModOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Mod - - @staticmethod - def get_override() -> List[str]: - return ['__mod__'] - - -class PowOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Pow - - @staticmethod - def get_override() -> List[str]: - return ['__pow__'] - - -class LShiftOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.LShift - - @staticmethod - def get_override() -> List[str]: - return ['__lshift__'] - - -class RShiftOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.RShift - - @staticmethod - def get_override() -> List[str]: - return ['__rshift__'] - - -class BitOrOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.BitOr - - @staticmethod - def get_override() -> List[str]: - return ['__or__'] - - -class BitXorOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.BitXor - - @staticmethod - def get_override() -> List[str]: - return ['__xor__'] - - -class BitAndOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.BitAnd - - @staticmethod - def get_override() -> List[str]: - return ['__and__'] - - -class FloorDivOperator(BinaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.FloorDiv - - @staticmethod - def get_override() -> List[str]: - return ['__floordiv__'] - - -class AndOperator(BooleanOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.And - - -class OrOperator(BooleanOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Or - - -class EqOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Eq - - @staticmethod - def get_override() -> List[str]: - return ['__eq__'] - - -class NotEqOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.NotEq - - @staticmethod - def get_override() -> List[str]: - return ['__ne__'] - - -class LtOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Lt - - @staticmethod - def get_override() -> List[str]: - return ['__lt__'] - - -class LtEOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.LtE - - @staticmethod - def get_override() -> List[str]: - return ['__le__'] - - -class GtOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Gt - - @staticmethod - def get_override() -> List[str]: - return ['__gt__'] - - -class GtEOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.GtE - - @staticmethod - def get_override() -> List[str]: - return ['__ge__'] - - -class IsOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Is - - -class IsNotOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.IsNot - - -class InOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.In - - -class NotInOperator(CompareOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.NotIn - - -class InvertOperator(UnaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Invert - - @staticmethod - def get_override() -> List[str]: - return ['__inv__', '__invert__'] - - -class NotOperator(UnaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.Not - - @staticmethod - def get_override() -> List[str]: - return ['__not__'] - - -class UAddOperator(UnaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.UAdd - - @staticmethod - def get_override() -> List[str]: - return ['__pos__'] - - -class USubOperator(UnaryOperator): - @staticmethod - def get_ast_type() -> Type[ast.AST]: - return ast.UAdd - - @staticmethod - def get_override() -> List[str]: - return ['__neg__'] - - -built_in_operators = inspect.getmembers(sys.modules[__name__], - lambda x: inspect.isclass(x) and issubclass(x, BuiltInOperator) and - x is not BuiltInOperator) - -number_precedence = [Complex, Float, Integer, Boolean] - - -def get_built_in_operator(t: ast.AST) -> Optional[BuiltInOperator]: - bio = [x for x in built_in_operators if type(t) is x[1].get_ast_type()] - assert len(bio) <= 1 - if len(bio) == 0: - return None - return bio[0][1]() diff --git a/plugins/python/parser/src/my_ast/built_in_types.py b/plugins/python/parser/src/my_ast/built_in_types.py deleted file mode 100644 index cfbe74256..000000000 --- a/plugins/python/parser/src/my_ast/built_in_types.py +++ /dev/null @@ -1,174 +0,0 @@ -import typing -from typing import Optional, Any - -from my_ast.base_data import TypeDeclaration -from my_ast.common.file_position import FilePosition - - -class BuiltIn(TypeDeclaration): - def __init__(self, name: str): - super().__init__(name, "builtins." + name, FilePosition.get_empty_file_position()) - - def get_type_repr(self) -> str: - return self.name - - def __hash__(self): - return hash(self.name) - - def __eq__(self, other): - return isinstance(other, type(self)) and self.name == other.name - - def __ne__(self, other): - return not self.__eq__(other) - - -class GenericType: - def __init__(self, types: Optional[typing.Set[Any]] = None): - if types is None: - self.types = set() - else: - self.types = types - - def add_type(self, new_type): - self.types.add(new_type) - - -class GenericBuiltInType(BuiltIn, GenericType): - def __init__(self, name: str, types: Optional[typing.Set[Any]] = None): - BuiltIn.__init__(self, name) - GenericType.__init__(self, types) - - def get_type_repr(self) -> str: - return self.name + '<' + ','.join({x.get_type_repr() for x in self.types}) + '>' - - -class Boolean(BuiltIn): - def __init__(self): - super().__init__('bool') - - -class Integer(BuiltIn): - def __init__(self): - super().__init__('int') - - -class Float(BuiltIn): - def __init__(self): - super().__init__('float') - - -class Complex(BuiltIn): - def __init__(self): - super().__init__('complex') - - -class String(BuiltIn): - def __init__(self): - super().__init__('str') - - -class List(GenericBuiltInType): - def __init__(self, types: Optional[typing.Set[Any]] = None): - super().__init__('list', types) - - -class Tuple(GenericBuiltInType): - def __init__(self, types: Optional[typing.Set[Any]] = None): - super().__init__('tuple', types) - - -class Range(BuiltIn): - def __init__(self): - super().__init__('range') - - -class Bytes(BuiltIn): - def __init__(self): - super().__init__('bytes') - - -class ByteArray(BuiltIn): - def __init__(self): - super().__init__('bytearray') - - -class MemoryView(BuiltIn): - def __init__(self): - super().__init__('memoryview') - - -class Set(GenericBuiltInType): - def __init__(self, types: Optional[typing.Set[Any]] = None): - super().__init__('set', types) - - -class FrozenSet(GenericBuiltInType): - def __init__(self, types: Optional[typing.Set[Any]] = None): - super().__init__('frozenset', types) - - -class Dictionary(GenericBuiltInType): - def __init__(self, types: Optional[typing.Set[Any]] = None): - super().__init__('dict', types) - - -class RangeType(BuiltIn): # generic? (implement __index__ method) - def __init__(self): - super().__init__('range') - - -class NoneType(BuiltIn): - def __init__(self): - super().__init__('None') - - -class EllipsisType(BuiltIn): - def __init__(self): - super().__init__('Ellipsis') - - -class Slice(BuiltIn): - def __init__(self): - super().__init__('slice') - - -class Module(BuiltIn): - def __init__(self): - super().__init__('module') - - -class Function(BuiltIn): - def __init__(self): - super().__init__('function') - - -class Method(BuiltIn): - def __init__(self): - super().__init__('method') - - -class Lambda(BuiltIn): - def __init__(self): - super().__init__('lambda') - - -class Generator(BuiltIn): - def __init__(self): - super().__init__('generator') - - -class Type(BuiltIn): - def __init__(self): - super().__init__('type') - - -class Object(BuiltIn): - def __init__(self): - super().__init__('object') - - -class File(BuiltIn): - def __init__(self): - super().__init__('file') - -# others: contextmanager, code (~str?), NotImplemented, internal diff --git a/plugins/python/parser/src/my_ast/class_data.py b/plugins/python/parser/src/my_ast/class_data.py deleted file mode 100644 index 1be824f2e..000000000 --- a/plugins/python/parser/src/my_ast/class_data.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import List - -from my_ast.common.file_position import FilePosition -from my_ast.base_data import TypeDeclaration, ImportedDeclaration, DocumentedType -import my_ast.function_data as fd -import my_ast.variable_data as vd -from my_ast.persistence.base_dto import UsageDTO - -from my_ast.persistence.class_dto import ClassDeclarationDTO, ClassMembersDTO - - -# TODO: @classmethod -class ClassDeclaration(TypeDeclaration, DocumentedType): - def __init__(self, name: str, qualified_name: str, position: FilePosition, base_classes: List['ClassDeclaration'], - documentation: str): - TypeDeclaration.__init__(self, name, qualified_name, position) - DocumentedType.__init__(self, documentation) - self.base_classes: List[ClassDeclaration] = base_classes - self.methods: List[fd.FunctionDeclaration] = [] - self.static_methods: List[fd.StaticFunctionDeclaration] = [] - self.attributes: List[vd.VariableDeclaration] = [] - self.static_attributes: List[vd.StaticVariableDeclaration] = [] - self.classes: List[ClassDeclaration] = [] - - def create_dto(self) -> ClassDeclarationDTO: - usages = [] - for usage in self.usages: - usages.append(UsageDTO(usage.file_position)) - types = set() - base_classes = set() - for base_class in self.base_classes: - base_classes.add(base_class.qualified_name) - members = ClassMembersDTO() - for m in self.methods: - members.methods.append(m.qualified_name) - for sm in self.static_methods: - members.static_methods.append(sm.qualified_name) - for a in self.attributes: - members.attributes.append(a.qualified_name) - for sa in self.static_attributes: - members.static_attributes.append(sa.qualified_name) - for c in self.classes: - members.classes.append(c.qualified_name) - return ClassDeclarationDTO(self.name, self.qualified_name, self.file_position, - types, usages, base_classes, members, self.documentation) - - def get_type_repr(self) -> str: - return self.name - - -class ImportedClassDeclaration(ClassDeclaration, ImportedDeclaration[ClassDeclaration]): - def __init__(self, name: str, pos: FilePosition, class_declaration: ClassDeclaration): - ClassDeclaration.__init__(self, name, "", pos, class_declaration.base_classes, "") - ImportedDeclaration.__init__(self, class_declaration) - self.type = {class_declaration} diff --git a/plugins/python/parser/src/my_ast/class_init_declaration.py b/plugins/python/parser/src/my_ast/class_init_declaration.py deleted file mode 100644 index 170e892fd..000000000 --- a/plugins/python/parser/src/my_ast/class_init_declaration.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import List, Union, Set, Optional - -import my_ast.base_data as data -from my_ast.built_in_types import BuiltIn -from my_ast.class_data import ClassDeclaration -from my_ast.common.file_position import FilePosition -from my_ast.function_data import FunctionDeclaration, FunctionParameter - - -class ClassInitDeclaration(FunctionDeclaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, params: List[FunctionParameter], - class_declaration: ClassDeclaration, - func_type: Optional[Set[Union[data.Declaration, BuiltIn]]] = None): - super().__init__(name, qualified_name, pos, params, "", - func_type if func_type is not None else {class_declaration}) - self.class_declaration = class_declaration diff --git a/plugins/python/parser/src/my_ast/class_preprocessor.py b/plugins/python/parser/src/my_ast/class_preprocessor.py deleted file mode 100644 index 4d2682585..000000000 --- a/plugins/python/parser/src/my_ast/class_preprocessor.py +++ /dev/null @@ -1,84 +0,0 @@ -import ast -from typing import List, Dict - -from my_ast.base_data import Declaration -from my_ast.common.utils import has_attr -from my_ast.preprocessed_data import PreprocessedDeclaration -from my_ast.preprocessed_function import PreprocessedFunction -from my_ast.preprocessed_variable import PreprocessedVariable - - -class PreprocessedClass(PreprocessedDeclaration): - def __init__(self, name: str): - super().__init__(name) - self.methods: List[PreprocessedFunction] = [] - self.attributes: List[PreprocessedVariable] = [] - self.classes: List[PreprocessedClass] = [] - self.type_usages: List[Declaration] = [] - - def append_method(self, name: str): - self.methods.append(PreprocessedFunction(name)) - - def append_attribute(self, name: str): - self.attributes.append(PreprocessedVariable(name)) - - def append_class(self, nested_class: 'PreprocessedClass'): # 'type' -> in case of type(self) - self.classes.append(nested_class) - - -class PreprocessedClassCollector(ast.NodeVisitor): - def __init__(self): - self.classes: List[PreprocessedClass] = [] - self.class_list: List[int] = [] - self.class_nest_class_map: Dict[PreprocessedClass, List[ast.ClassDef]] = {} - - def append_class(self, node: ast.ClassDef): - self.class_list.append(len(self.classes)) - preprocessed_class = PreprocessedClass(node.name) - self.classes.append(preprocessed_class) - self.handle_nested_class(node, preprocessed_class) - for member in node.body: - if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): - self.append_method(member) - elif isinstance(member, ast.Assign): - self.append_attribute(member) - elif isinstance(member, ast.ClassDef): - self.append_nested_class(preprocessed_class, member) - elif isinstance(member, ast.Pass): - pass - elif isinstance(member, ast.Expr) and hasattr(member, 'value') and isinstance(member.value, ast.Constant) \ - and hasattr(member.value, "value") and isinstance(member.value.value, str): - pass # TODO: documentation comment - else: - assert False, "Unknown class member: " + str(type(member)) - - def handle_nested_class(self, node: ast.ClassDef, preprocessed_class: PreprocessedClass): - for parent_class in self.class_nest_class_map: - if node in self.class_nest_class_map[parent_class]: - parent_class.append_class(preprocessed_class) - self.class_nest_class_map[parent_class].remove(node) - - def class_processed(self): - del self.class_list[-1] - - def append_method(self, node: ast.FunctionDef): - self.classes[self.class_list[-1]].append_method(node.name) - if node.name == '__init__': - self.visit(node) - - def append_attribute(self, node: ast.Assign): - for attribute in node.targets: - if hasattr(attribute, 'id'): # class variable - self.classes[self.class_list[-1]].append_attribute(attribute.id) - elif hasattr(attribute, 'attr') and has_attr(attribute, 'value.id') and attribute.value.id == 'self': - self.classes[self.class_list[-1]].append_attribute(attribute.attr) - - def append_nested_class(self, node: PreprocessedClass, member: ast.ClassDef): - if node in self.class_nest_class_map: - self.class_nest_class_map[node].append(member) - else: - self.class_nest_class_map[node] = [member] - - def visit_Assign(self, node: ast.Assign): - self.append_attribute(node) - self.generic_visit(node) diff --git a/plugins/python/parser/src/my_ast/common/__init__.py b/plugins/python/parser/src/my_ast/common/__init__.py deleted file mode 100644 index 1df5a03e2..000000000 --- a/plugins/python/parser/src/my_ast/common/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ["hashable_list", "file_position", "parser_tree", "position", "unique_list", "utils"] diff --git a/plugins/python/parser/src/my_ast/common/file_position.py b/plugins/python/parser/src/my_ast/common/file_position.py deleted file mode 100644 index d653e9dd9..000000000 --- a/plugins/python/parser/src/my_ast/common/file_position.py +++ /dev/null @@ -1,20 +0,0 @@ -from pathlib import PurePath -from typing import Optional - -from my_ast.common.position import Range - - -class FilePosition: - def __init__(self, file: Optional[PurePath], r: Range): - self.file: Optional[PurePath] = file # Optional -> builtins - self.range: Range = r - - def __str__(self): - return "File: " + str(self.file) + " - " + str(self.range) - - def __repr__(self): - return self.__str__() - - @staticmethod - def get_empty_file_position(): - return FilePosition(None, Range.get_empty_range()) diff --git a/plugins/python/parser/src/my_ast/common/hashable_list.py b/plugins/python/parser/src/my_ast/common/hashable_list.py deleted file mode 100644 index 45d6deae5..000000000 --- a/plugins/python/parser/src/my_ast/common/hashable_list.py +++ /dev/null @@ -1,31 +0,0 @@ -from collections import Counter -from typing import List, TypeVar, Generic - -T = TypeVar('T') - - -class HashableList(Generic[T], List[T]): - def __hash__(self): - return hash(e for e in self) - - def __eq__(self, other): - return isinstance(other, type(self)) and Counter(self) == Counter(other) - - def __ne__(self, other): - return not self.__eq__(other) - - -class OrderedHashableList(Generic[T], List[T]): - def __hash__(self): - return hash(e for e in self) - - def __eq__(self, other): - if not isinstance(other, type(self)) or len(other) != len(self): - return False - for i in range(0, len(self)): - if self[i] != other[i]: - return False - return True - - def __ne__(self, other): - return not self.__eq__(other) diff --git a/plugins/python/parser/src/my_ast/common/location.py b/plugins/python/parser/src/my_ast/common/location.py deleted file mode 100644 index b6c9a14dd..000000000 --- a/plugins/python/parser/src/my_ast/common/location.py +++ /dev/null @@ -1,8 +0,0 @@ -from my_ast.common.position import Position - - -class Location: - def __init__(self, path: str, file: str, pos: Position): - self.path = path - self.file = file - self.position = pos diff --git a/plugins/python/parser/src/my_ast/common/parser_tree.py b/plugins/python/parser/src/my_ast/common/parser_tree.py deleted file mode 100644 index cc0d60dab..000000000 --- a/plugins/python/parser/src/my_ast/common/parser_tree.py +++ /dev/null @@ -1,30 +0,0 @@ -import ast - - -class ParserTreeNode: - def __init__(self, node, parent: 'ParserTreeNode'): - self.node = node - self.parent = parent - self.children = [] - self.process_children() - - def process_children(self): - for child in ast.iter_child_nodes(self.node): - self.children.append(ParserTreeNode(child, self)) - - -class ParserTree: - def __init__(self, node): - self.root = ParserTreeNode(node, None) - - def find_node(self, node) -> ParserTreeNode: - return self.find_node_in_parent(self.root, node) - - def find_node_in_parent(self, parent: ParserTreeNode, node) -> ParserTreeNode: - if parent.node is node: - return parent - for child in parent.children: - n = self.find_node_in_parent(child, node) - if n is not None: - return n - return None diff --git a/plugins/python/parser/src/my_ast/common/position.py b/plugins/python/parser/src/my_ast/common/position.py deleted file mode 100644 index 1f7f06187..000000000 --- a/plugins/python/parser/src/my_ast/common/position.py +++ /dev/null @@ -1,36 +0,0 @@ -class Position: - def __init__(self, line: int, col: int): - self.line = line - self.column = col - - def __str__(self): - return "line: " + str(self.line) + ", column: " + str(self.column) - - def __repr__(self): - return self.__str__() - - def __eq__(self, other): - return self.line == other.line and self.column == other.column - - def __ne__(self, other): - return not self.__eq__(other) - - @staticmethod - def get_empty_position(): - return Position(0, 0) - - -class Range: - def __init__(self, start_pos: Position, end_pos: Position): - self.start_position = start_pos - self.end_position = end_pos - - def __str__(self): - return "Start position: " + str(self.start_position) + " - End position: " + str(self.end_position) - - def __repr__(self): - return self.__str__() - - @staticmethod - def get_empty_range(): - return Range(Position.get_empty_position(), Position.get_empty_position()) diff --git a/plugins/python/parser/src/my_ast/common/unique_list.py b/plugins/python/parser/src/my_ast/common/unique_list.py deleted file mode 100644 index 952f707ae..000000000 --- a/plugins/python/parser/src/my_ast/common/unique_list.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import TypeVar, List, Iterable - -T = TypeVar('T') - - -class UniqueList(List[T]): - def __init__(self, seq=()): - super().__init__() - self.extend(seq) - - def append(self, obj: T) -> None: - if obj in self: - self.remove(obj) - super().append(obj) - - def extend(self, iterable: Iterable[T]) -> None: - for i in iterable: - self.append(i) - - def insert(self, index: int, obj: T) -> None: - if obj in self: - self.remove(obj) - super().insert(index, obj) diff --git a/plugins/python/parser/src/my_ast/common/utils.py b/plugins/python/parser/src/my_ast/common/utils.py deleted file mode 100644 index db3519ea8..000000000 --- a/plugins/python/parser/src/my_ast/common/utils.py +++ /dev/null @@ -1,18 +0,0 @@ -import ast - -from my_ast.common.position import Range, Position - - -def has_attr(obj, attrs) -> bool: - for attr in attrs.split("."): - if hasattr(obj, attr): - obj = getattr(obj, attr) - else: - return False - return True - - -def create_range_from_ast_node(node: ast.AST) -> Range: - start_pos = Position(node.lineno, node.col_offset) - end_pos = Position(node.end_lineno, node.end_col_offset) - return Range(start_pos, end_pos) diff --git a/plugins/python/parser/src/my_ast/db_test.log b/plugins/python/parser/src/my_ast/db_test.log deleted file mode 100644 index 9976880ff..000000000 --- a/plugins/python/parser/src/my_ast/db_test.log +++ /dev/null @@ -1,27 +0,0 @@ - -FILE: __init__ -========================================= -Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] - -FILE: asdf -========================================= - -FILE: qwer -========================================= -Func: g (line: 1, column: 0, length: 1) {global - } [] - -FILE: asdf -========================================= -Var: a (line: 4, column: 0, length: 1) {function - f} [Placeholder] -Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] -Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] -Var: b (line: 4, column: 0, length: 1) {function - f} [Placeholder] -Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] -Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] -Func: g (line: 0, column: 0, length: 0) {global - } [] -Func Usage: g (line: 7, column: 4, length: 1) [] -Func: f (line: 4, column: 0, length: 1) {global - } [str] - -FILE: __init__ -========================================= -Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] diff --git a/plugins/python/parser/src/my_ast/file_info.py b/plugins/python/parser/src/my_ast/file_info.py deleted file mode 100644 index a79588140..000000000 --- a/plugins/python/parser/src/my_ast/file_info.py +++ /dev/null @@ -1,59 +0,0 @@ -from enum import Enum, unique, auto -from pathlib import PurePath, Path -from typing import Optional - -from my_ast.persistence.file_content_dto import FileContentDTO -from my_ast.persistence.file_dto import FileDTO -from my_ast.preprocessed_file import PreprocessedFile -from my_ast.symbol_collector import SymbolCollector - - -@unique -class ProcessStatus(Enum): - WAITING = auto() - PREPROCESSED = auto() - PROCESSED = auto() - - -# TODO: expr_1; import; expr_2; - correct script -> not a problem -class FileInfo: - def __init__(self, file: str, path: Optional[PurePath]): # TODO: remove optional - self.file: str = file - self.path: Optional[PurePath] = path - self.symbol_collector: Optional[SymbolCollector] = None - self.preprocessed_file: PreprocessedFile = PreprocessedFile() - self.status: ProcessStatus = ProcessStatus.WAITING - - def preprocess_file(self, tree) -> None: - self.preprocessed_file.visit(tree) - self.status = ProcessStatus.PREPROCESSED - - def set_variable_collector(self, variable_collector: SymbolCollector) -> None: - self.symbol_collector = variable_collector - self.status = ProcessStatus.PROCESSED - - def create_dto(self) -> FileDTO: - file_dto = FileDTO() - file_dto.path = str(self.path) - file_dto.file_name = self.file - file_dto.timestamp = Path(self.path).stat().st_mtime - file_dto.content = FileContentDTO(self.get_content(self.path)) - file_dto.parent = str(self.path.parent) - file_dto.parse_status = self.get_parse_status(self.status) - file_dto.documentation = self.preprocessed_file.documentation - return file_dto - - @staticmethod - def get_content(file: PurePath) -> str: - with open(file) as f: - return f.read() - - @staticmethod - def get_parse_status(status: ProcessStatus) -> int: - if status == ProcessStatus.WAITING: - return 0 - elif status == ProcessStatus.PREPROCESSED: - return 1 - elif status == ProcessStatus.PROCESSED: - return 2 - assert False diff --git a/plugins/python/parser/src/my_ast/function_data.py b/plugins/python/parser/src/my_ast/function_data.py deleted file mode 100644 index 21a9ec5ad..000000000 --- a/plugins/python/parser/src/my_ast/function_data.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import List, Optional, Set, Union - -from my_ast.common.file_position import FilePosition -import my_ast.base_data as data -from my_ast.persistence.base_dto import UsageDTO -from my_ast.persistence.function_dto import FunctionDeclarationDTO -from my_ast.type_data import DeclarationType - - -class FunctionParameter: - def __init__(self, name: str, func_type: str = None): - self.name: str = name - self.type: Optional[str] = func_type - - -class FunctionDeclaration(data.Declaration, data.DocumentedType): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, params: List[FunctionParameter], - documentation: str, func_type: Optional[Set[Union[DeclarationType]]] = None): - data.Declaration.__init__(self, name, qualified_name, pos, func_type) - data.DocumentedType.__init__(self, documentation) - self.parameters: List[FunctionParameter] = params - - def create_dto(self) -> FunctionDeclarationDTO: - usages = [] - for usage in self.usages: - usages.append(UsageDTO(usage.file_position)) - types = set() - for t in self.type: - types.add(t.qualified_name) - return FunctionDeclarationDTO(self.name, self.qualified_name, self.file_position, - types, usages, self.documentation) - - -class StaticFunctionDeclaration(FunctionDeclaration): - pass - - -class ImportedFunctionDeclaration(FunctionDeclaration, data.ImportedDeclaration[FunctionDeclaration]): - def __init__(self, name: str, pos: FilePosition, func_declaration: FunctionDeclaration): - FunctionDeclaration.__init__(self, name, "", pos, func_declaration.parameters, "", func_declaration.type) - data.ImportedDeclaration.__init__(self, func_declaration) diff --git a/plugins/python/parser/src/my_ast/function_symbol_collector.py b/plugins/python/parser/src/my_ast/function_symbol_collector.py deleted file mode 100644 index 2bbe13519..000000000 --- a/plugins/python/parser/src/my_ast/function_symbol_collector.py +++ /dev/null @@ -1,64 +0,0 @@ -import ast -from typing import Union, Any, List - -from my_ast.common.file_position import FilePosition -from my_ast.common.parser_tree import ParserTree -from my_ast.common.utils import create_range_from_ast_node -from my_ast.function_data import FunctionDeclaration -from my_ast.scope import FunctionScope -from my_ast.symbol_collector import SymbolCollector -from my_ast.symbol_collector_interface import IFunctionSymbolCollector -from my_ast.type_data import DeclarationType -from my_ast.type_deduction import TypeDeduction -from my_ast.variable_data import VariableDeclaration - - -class TemporaryScope(FunctionScope): - pass - - -class FunctionSymbolCollector(SymbolCollector, IFunctionSymbolCollector): - def __init__(self, symbol_collector: SymbolCollector, tree: Union[ast.FunctionDef, ast.AsyncFunctionDef], - arguments: List[DeclarationType]): - SymbolCollector.__init__(self, ParserTree(tree), symbol_collector.current_file, - symbol_collector.preprocessed_file, - symbol_collector.import_finder, - symbol_collector.function_symbol_collector_factory) - IFunctionSymbolCollector.__init__(self) - self.scope_manager = symbol_collector.scope_manager - self.type_deduction = TypeDeduction(self, self.scope_manager, self.function_symbol_collector_factory) - self.arguments = arguments - - # TODO: handle local functions - # TODO: circular import + placeholder? - def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: - r = create_range_from_ast_node(node) - self.function = FunctionDeclaration(node.name, - self.scope_manager.get_qualified_name_from_current_scope(node.name), - FilePosition(self.current_file, r), [], "") - # self.scope_manager.append_function_to_current_scope(self.function) - self.current_function_declaration.append(self.function) - - def action(): - self.collect_parameters(node) - self.generic_visit(node) - - self.scope_manager.with_scope(FunctionScope(node.name), action) - del self.current_function_declaration[-1] - - def collect_parameters(self, node: ast.FunctionDef) -> None: - arg_idx = 0 - for param in node.args.args: - if hasattr(param, 'arg'): # ast.Name - r = create_range_from_ast_node(node) - new_variable = VariableDeclaration(param.arg, - self.scope_manager.get_qualified_name_from_current_scope(param.arg), - FilePosition(self.current_file, r)) - if param.arg == 'self' and self.scope_manager.is_current_scope_method() and node.args.args[0] is param: - new_variable.type.add(self.scope_manager.get_current_class_declaration()) - else: - new_variable.type.update(self.type_deduction.get_current_type({self.arguments[arg_idx]})) - arg_idx += 1 - self.scope_manager.append_variable_to_current_scope(new_variable) - else: - assert False, "Parameter with unknown variable name" diff --git a/plugins/python/parser/src/my_ast/function_symbol_collector_factory.py b/plugins/python/parser/src/my_ast/function_symbol_collector_factory.py deleted file mode 100644 index 88d5234f6..000000000 --- a/plugins/python/parser/src/my_ast/function_symbol_collector_factory.py +++ /dev/null @@ -1,14 +0,0 @@ -import ast -from abc import ABC, abstractmethod -from typing import Union, List - -from my_ast.symbol_collector_interface import SymbolCollectorBase, IFunctionSymbolCollector -from my_ast.type_data import DeclarationType - - -class FunctionSymbolCollectorFactory(ABC): - @abstractmethod - def get_function_symbol_collector(self, symbol_collector: SymbolCollectorBase, - func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], - arguments: List[DeclarationType]) -> IFunctionSymbolCollector: - pass diff --git a/plugins/python/parser/src/my_ast/import_finder.py b/plugins/python/parser/src/my_ast/import_finder.py deleted file mode 100644 index 93eb5238e..000000000 --- a/plugins/python/parser/src/my_ast/import_finder.py +++ /dev/null @@ -1,27 +0,0 @@ -from abc import ABC, abstractmethod -from pathlib import PurePath -from typing import Optional - - -class ImportFinder(ABC): - # Optional[FileInfo] - circular import - @abstractmethod - def get_file_by_location(self, location: PurePath) -> Optional: - pass - - # Optional[GlobalScope] - circular import - def get_global_scope_by_location(self, location: PurePath) -> Optional: - file = self.get_file_by_location(location) - if file is not None and file.symbol_collector is not None: - assert file.symbol_collector.scope_manager.get_size() == 1 - return file.symbol_collector.scope_manager.get_global_scope() - elif file is not None: - return file.preprocessed_file - return None - - def get_scope_manager_by_location(self, location: PurePath) -> Optional: - file = self.get_file_by_location(location) - if file is not None and file.symbol_collector is not None: - assert file.symbol_collector.scope_manager.get_size() == 1 - return file.symbol_collector.scope_manager - return None diff --git a/plugins/python/parser/src/my_ast/import_preprocessor.py b/plugins/python/parser/src/my_ast/import_preprocessor.py deleted file mode 100644 index 1bbdecf26..000000000 --- a/plugins/python/parser/src/my_ast/import_preprocessor.py +++ /dev/null @@ -1,129 +0,0 @@ -import ast -from pathlib import PurePath -from typing import Optional, List -from importlib import util # python 3.4 or above - -from my_ast.common.position import Range -from my_ast.common.utils import create_range_from_ast_node - - -class FileDependency: - def __init__(self, location: PurePath, module: str, alias: str = None): - self.location = location - self.module: str = module - self.alias: Optional[str] = alias - - def get_file(self) -> str: - module_parts = self.module.split('.') - return module_parts[-1] + '.py' - - -class ModuleImport: - def __init__(self, name: str, alias: Optional[str] = None): - self.name: str = name - self.alias: Optional[str] = alias - - -class SymbolImport: - def __init__(self, name: str, alias: Optional[str] = None): - self.name: str = name - self.alias: Optional[str] = alias - - def is_all_imported(self) -> bool: - return self.name == '*' - - -class Import: - def __init__(self, path: List[str], module: ModuleImport, imported: List[SymbolImport], - location: Optional[PurePath], r: Range, is_local: bool = False): - self.path: List[str] = path - self.module: ModuleImport = module - self.imported: List[SymbolImport] = imported - self.location: Optional[PurePath] = location - self.range = r - self.is_local = is_local - - def is_module_import(self): - return len(self.imported) == 0 - - -# TODO: need locations of files, and check if modules are among them? -# TODO: handle relative import: -# '.': from the package (parent directory) of the current module (from .'package' import 'module') -# from the __init__ (from . import 'symbol') -# '..': from the grandparent directory of the current module (and so on) -class ImportTable: - def __init__(self): - self.modules: List[Import] = [] - - def append_import(self, node: ast.Import, is_local: bool = False): - self.modules.extend(self.convert_ast_import_to_import(node, is_local)) - - def append_import_from(self, node: ast.ImportFrom, is_local: bool = False): - self.modules.extend(self.convert_ast_import_from_to_import(node, is_local)) - - @staticmethod - def convert_ast_import_to_import(node: ast.Import, is_local: bool = False) -> List[Import]: - imports = [] - for module in node.names: - module_parts = module.name.split('.') - module_name = module_parts[-1] - module_parts.remove(module_name) - r = create_range_from_ast_node(node) - imports.append(ImportTable.create_import(module_parts, module_name, module.asname, [], r, is_local)) - return imports - - @staticmethod - def convert_ast_import_from_to_import(node: ast.ImportFrom, is_local: bool = False) -> List[Import]: - assert len(node.names) > 0 - is_module = True - try: - if util.find_spec(node.module + '.' + node.names[0].name) is None: - is_module = False - except (AttributeError, ModuleNotFoundError): # v3.7: before: AttributeError, after: ModuleNotFoundError - is_module = False - imports = [] - r = create_range_from_ast_node(node) - if is_module: # modules - for module in node.names: - imports.append( - ImportTable.create_import(node.module.split('.'), module.name, module.asname, [], r, is_local)) - else: # variables, functions - imported = [] - for module in node.names: - imported.append(SymbolImport(module.name, module.asname)) - module_parts = node.module.split('.') - module_name = module_parts[-1] - module_parts.remove(module_name) - imports.append(ImportTable.create_import(module_parts, module_name, None, imported, r, is_local)) - return imports - - @staticmethod - def create_import(path: List[str], name: str, alias: Optional[str], imported: List[SymbolImport], - r: Range, is_local: bool = False) -> Import: - location = None - module = path - module.append(name) - try: - spec = util.find_spec('.'.join(module)) - if spec is None: - assert False, "Cannot find module: " + '.'.join(module) - elif spec.has_location: - location = spec.origin - except (AttributeError, ModuleNotFoundError): # v3.7: before: AttributeError, after: ModuleNotFoundError - assert False, "Cannot find module: " + '.'.join(module) - optional_location = None - if location is not None: - optional_location = PurePath(location) - return Import(path, ModuleImport(name, alias), imported, optional_location, r, is_local) - - def get_dependencies(self) -> List[FileDependency]: - dependencies = [] - for module in self.modules: - if len(module.path) == 0: - dependencies.append(FileDependency(module.location, module.module.name, module.module.alias)) - else: - dependencies.append( - FileDependency( - module.location, '.'.join(module.path) + '.' + module.module.name, module.module.alias)) - return dependencies diff --git a/plugins/python/parser/src/my_ast/logger.py b/plugins/python/parser/src/my_ast/logger.py deleted file mode 100644 index a4621e7e2..000000000 --- a/plugins/python/parser/src/my_ast/logger.py +++ /dev/null @@ -1,16 +0,0 @@ -import pathlib -import logging - -directory = pathlib.Path(__file__).resolve().parent - -logger = logging.getLogger('test') -logger.setLevel(logging.DEBUG) -file_handler = logging.FileHandler(str(directory) + '/test.log', 'w') -file_handler.setLevel(logging.DEBUG) -logger.addHandler(file_handler) - -db_logger = logging.getLogger('db_test') -db_logger.setLevel(logging.DEBUG) -db_file_handler = logging.FileHandler(str(directory) + '/db_test.log', 'w') -db_file_handler.setLevel(logging.DEBUG) -db_logger.addHandler(db_file_handler) diff --git a/plugins/python/parser/src/my_ast/main.py b/plugins/python/parser/src/my_ast/main.py deleted file mode 100644 index 6ebef57a1..000000000 --- a/plugins/python/parser/src/my_ast/main.py +++ /dev/null @@ -1,43 +0,0 @@ -import ast -import os -from pathlib import PurePath -from typing import Optional - -from my_ast.common.parser_tree import ParserTree -from my_ast.file_info import FileInfo -from my_ast.import_finder import ImportFinder -from my_ast.parse_exception import ParseException -from my_ast.parser import Parser -from my_ast.symbol_collector import SymbolCollector - - -def main(): - # with open("../test_dir/test_file_2.py", "r") as source: - # tree = ast.parse(source.read()) - # - # pt = ParserTree(tree) - # - # fi = FileInfo("test_file_2.py", PurePath("../test_dir/test_file_2.py")) - # - # class ImportFinderHelper(ImportFinder): - # def get_file_by_location(self, location: PurePath) -> Optional: - # return None - # - # vdc = SymbolCollector(pt, fi.preprocessed_file, ImportFinderHelper()) - # vdc.collect_symbols() - - def directory_exception(path: PurePath) -> bool: - directory = os.path.basename(os.path.normpath(str(path))) - return directory.startswith('.') or directory == 'venv' - - def file_exception(path: PurePath) -> bool: - return False - - exception = ParseException(directory_exception, file_exception) - p = Parser(["F:/ELTE/PythonPlugin"], exception) - p.parse() - pass - - -if __name__ == '__main__': - main() diff --git a/plugins/python/parser/src/my_ast/member_access_collector.py b/plugins/python/parser/src/my_ast/member_access_collector.py deleted file mode 100644 index 92a9fcbc9..000000000 --- a/plugins/python/parser/src/my_ast/member_access_collector.py +++ /dev/null @@ -1,210 +0,0 @@ -import ast -from typing import Optional, List, Any, Union - - -class MemberAccessCollector(ast.NodeVisitor): - class MemberData: - def __init__(self, name: Optional[str]): - self.name: str = name - - class ConstantData(MemberData): - pass - - class AttributeData(MemberData): - pass - - class MethodData(MemberData): - def __init__(self, name: Optional[str], args): - super().__init__(name) - self.arguments = args - - class ReturnValueData(MemberData): - def __init__(self, name: Optional[str], member_data, node: ast.AST): - super().__init__(name) - self.return_of: MemberAccessCollector.MemberData = member_data - self.node = node - - class SubscriptData(MemberData): - class Index: - def __init__(self, idx): - self.idx = idx - - class Slice: - def __init__(self, lower, upper, step): - self.lower = lower - self.upper = upper - self.step = step - - def __init__(self, name: Optional[str], sub_slice: List[Union[Index, Slice]]): - super().__init__(name) - self.slice: List[Union[MemberAccessCollector.SubscriptData.Index, - MemberAccessCollector.SubscriptData.Slice]] = sub_slice - - class OperatorData(MemberData): - def __init__(self, name: Optional[str], node: Union[ast.BoolOp, ast.BinOp, ast.UnaryOp, ast.Compare]): - super().__init__(name) - - class LambdaData(MemberData): - def __init__(self, name: Optional[str], node: ast.Lambda): - super().__init__(name) - - def __init__(self, member: Union[ast.Call, ast.Attribute]): - self.call_list: List[MemberAccessCollector.MemberData] = [] - self.arguments = [] - self.last = False - self.visit(member) - - def generic_visit(self, node: ast.AST): - if self.last: - return - if not isinstance(node, ast.Await): # await must be skipped, and process the callable - print('Unhandled node type: ' + str(type(node))) - ast.NodeVisitor.generic_visit(self, node) - - def visit_Call(self, node: ast.Call) -> Any: - if self.last: - return - - def process_call(value: ast.AST): - if isinstance(value, (ast.Name, ast.Attribute, ast.Lambda)): - self.call_list.append(self.MethodData(None, self.collect_arguments(node))) - elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): - self.call_list.append( - self.ReturnValueData(None, self.MethodData(None, self.collect_arguments(node)), value)) - elif isinstance(value, ast.Await): - process_call(value.value) - else: - assert False - process_call(node.func) - ast.NodeVisitor.generic_visit(self, node) - - def visit_Subscript(self, node: ast.Subscript) -> Any: - if self.last: - return - - def process_subscript(value: ast.AST): - if isinstance(value, (ast.Name, ast.Attribute)): - self.call_list.append(self.SubscriptData(None, self.collect_slice(node))) - elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): - self.call_list.append( - self.ReturnValueData(None, self.SubscriptData(None, self.collect_slice(node)), value)) - elif isinstance(value, ast.Await): - process_subscript(value.value) - else: - assert False, "Not subscriptable type: " + str(type(value)) - process_subscript(node.value) - ast.NodeVisitor.generic_visit(self, node) - - def visit_Attribute(self, node: ast.Attribute) -> Any: - if self.last: - return - if len(self.call_list) > 0 and self.call_list[-1].name is None: - self.call_list[-1].name = node.attr - else: - self.call_list.append(self.AttributeData(node.attr)) - ast.NodeVisitor.generic_visit(self, node) - - def visit_Name(self, node: ast.Name) -> Any: - if self.last: - return - if self.call_list[-1].name is None: - self.call_list[-1].name = node.id - else: - self.call_list.append(self.AttributeData(node.id)) - self.last = True - - def visit_Constant(self, node: ast.Constant) -> Any: - if self.last: - return - if self.call_list[-1].name is None: - self.call_list[-1].name = node.value - else: - value = node.value - if isinstance(value, str): - value = '"' + value + '"' - self.call_list.append(self.ConstantData(value)) - self.last = True - - def visit_Lambda(self, node: ast.Lambda) -> Any: - if self.last: - return - if self.call_list[-1].name is None: - self.call_list[-1].name = 'lambda' - else: - self.call_list.append(self.LambdaData('lambda', node)) - self.last = True - - def visit_BinOp(self, node: ast.BinOp) -> Any: - self.last = True - - def visit_UnaryOp(self, node: ast.UnaryOp) -> Any: - self.last = True - - def visit_BoolOp(self, node: ast.BoolOp) -> Any: - self.last = True - - def visit_Compare(self, node: ast.Compare) -> Any: - self.last = True - - # TODO: is it necessary? - @staticmethod - def collect_arguments(call: ast.Call) -> List: - arguments = [] - for arg in call.args: - arguments.append(arg) - return arguments - - @staticmethod - def collect_slice(subscript: ast.Subscript) -> List[Union[SubscriptData.Index, SubscriptData.Slice]]: - # TODO: in python 3.9 this must be changed (ast.Index, ast.ExtSlice are deprecated) - sub_slice = [] - - def process_slice(node: (ast.Index, ast.Slice, ast.ExtSlice)): - if isinstance(node, ast.Index): - sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) - elif isinstance(node, ast.Slice): - sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) - elif isinstance(node, ast.ExtSlice): # 3.9 -> ast.Tuple - for dim in node.dims: - process_slice(dim) - else: - assert False, "Unknown slice: " + str(type(node)) - - process_slice(subscript.slice) - return sub_slice - - @staticmethod - def is_callable_after_override(node: ast.AST) -> bool: - return isinstance(node, ast.BoolOp) or isinstance(node, ast.BinOp) or isinstance(node, ast.UnaryOp) or \ - isinstance(node, ast.Compare) - - -class MemberAccessCollectorIterator: - def __init__(self, member_access_collector: MemberAccessCollector): - self.mac = member_access_collector - self.index = len(self.mac.call_list) - - def __iter__(self): - self.index = len(self.mac.call_list) - return self - - def __next__(self): - if self.index <= 0: - raise StopIteration - self.index -= 1 - return self.mac.call_list[self.index] - - def get_current(self): - return self.mac.call_list[self.index] - - def get_first(self): - return self.mac.call_list[-1] - - def get_last(self): - return self.mac.call_list[0] - - def is_iteration_over(self) -> bool: - return self.index <= 0 - - def is_iteration_started(self) -> bool: - return self.index < len(self.mac.call_list) diff --git a/plugins/python/parser/src/my_ast/parse_exception.py b/plugins/python/parser/src/my_ast/parse_exception.py deleted file mode 100644 index 2ff8a6448..000000000 --- a/plugins/python/parser/src/my_ast/parse_exception.py +++ /dev/null @@ -1,20 +0,0 @@ -from pathlib import PurePath -from typing import Callable # python 3.5 or above - - -class ParseException: - def __init__(self, is_dictionary_exception: Callable[[PurePath], bool], - is_file_exception: Callable[[PurePath], bool]): - self.is_dictionary_exception = is_dictionary_exception - self.is_file_exception = is_file_exception - - def is_dictionary_exception(self, dictionary: PurePath) -> bool: - return self.is_dictionary_exception(dictionary) - - def is_file_exception(self, file: PurePath) -> bool: - return self.is_file_exception(file) - - -class DefaultParseException(ParseException): - def __init__(self): - super().__init__(lambda x: False, lambda x: False) diff --git a/plugins/python/parser/src/my_ast/parser.py b/plugins/python/parser/src/my_ast/parser.py deleted file mode 100644 index 8aa8472c8..000000000 --- a/plugins/python/parser/src/my_ast/parser.py +++ /dev/null @@ -1,127 +0,0 @@ -import ast -import os -import sys -from pathlib import PurePath -from typing import List, Optional, Union - -from my_ast.common.parser_tree import ParserTree -from my_ast.file_info import FileInfo, ProcessStatus -from my_ast.function_symbol_collector import FunctionSymbolCollector -from my_ast.function_symbol_collector_factory import FunctionSymbolCollectorFactory -from my_ast.import_finder import ImportFinder -from my_ast.logger import logger, db_logger -from my_ast.parse_exception import DefaultParseException, ParseException -from my_ast.persistence.persistence import model_persistence -from my_ast.scope import GlobalScope -from my_ast.symbol_collector import SymbolCollector - -# TODO: builtins - dir(builtins) -# TODO: change logging to DB storing -from my_ast.symbol_collector_interface import IFunctionSymbolCollector -from my_ast.type_data import DeclarationType - -current_file = "" # TODO: only for debug, remove it - - -class Parser(ast.NodeVisitor, ImportFinder, FunctionSymbolCollectorFactory): - def __init__(self, directories: List[str], exceptions: Optional[ParseException] = None): - self.directories: List[str] = directories - if exceptions is None: - exceptions = DefaultParseException() - self.exceptions: ParseException = exceptions - self.files: List[FileInfo] = [] - self.parsing_started_files = [] - self.collect_files() - - def collect_files(self): - for directory in self.directories: - sys.path.append(directory) - self.process_directory(PurePath(directory)) - - def process_directory(self, directory: PurePath): - sub_directories = [] - for element in os.listdir(str(directory)): - path = directory.joinpath(element) - if os.path.isfile(str(path)): - if self.exceptions.is_file_exception(path): - continue - if element.endswith(".py"): - self.files.append(FileInfo(element, path)) - elif os.path.isdir(str(path)): - if self.exceptions.is_dictionary_exception(path): - continue - sub_directories.append(path) - else: - assert False, "Unknown element (file, directory): " + str(path) - - for subdir in sub_directories: - self.process_directory(subdir) - - def parse(self) -> None: - for file_info in self.files: - if file_info.symbol_collector is None: - self.parse_file(file_info) - self.persist_global_scopes() - - def parse_file(self, file_info: FileInfo) -> None: - global current_file - current_file = file_info.file - if file_info.status != ProcessStatus.WAITING: - return - # if file_info in self.parsing_started_files: - # return - # self.parsing_started_files.append(file_info) - logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - db_logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - # TODO: fix this... - # if file_info.file[:-3] in builtin_module_names: # [:-3] - '.py' extension - # return # TODO: handle this case (+ deprecated modules!) - if file_info.path is None: - return - try: - with open(str(file_info.path), "r") as source: - tree = ast.parse(source.read()) - file_info.preprocess_file(tree) - for dependency in file_info.preprocessed_file.import_table.get_dependencies(): - dependency_file_info = [x for x in self.files if x.path == dependency.location] - if len(dependency_file_info) == 0: - pass - # TODO: do we need this? (eg. PYTHONPATH) - # new_file_info = FileInfo(file, dependency.location) - # self.files.append(new_file_info) - # self.parse_file(new_file_info) - # logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - elif len(dependency_file_info) == 1: - if dependency_file_info[0].status is ProcessStatus.WAITING: - self.parse_file(dependency_file_info[0]) - current_file = file_info.file - logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - db_logger. \ - debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - else: - assert False, 'Multiple file occurrence: ' + dependency.get_file() - sc = SymbolCollector(ParserTree(tree), file_info.path, file_info.preprocessed_file, self, self) - sc.collect_symbols() - file_info.set_variable_collector(sc) - - if model_persistence is not None: - model_persistence.persist_file(file_info.create_dto()) - except FileNotFoundError: - return - - def get_file_by_location(self, location: PurePath) -> Optional[FileInfo]: - for file in self.files: - if file.path == location: - return file - return None - - def persist_global_scopes(self): - for file_info in self.files: - if file_info.symbol_collector is not None: - assert isinstance(file_info.symbol_collector.scope_manager.get_current_scope(), GlobalScope) - file_info.symbol_collector.scope_manager.persist_current_scope() - - def get_function_symbol_collector(self, symbol_collector: SymbolCollector, - func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], - arguments: List[DeclarationType]) -> IFunctionSymbolCollector: - return FunctionSymbolCollector(symbol_collector, func_def, arguments) diff --git a/plugins/python/parser/src/my_ast/persistence/__init__.py b/plugins/python/parser/src/my_ast/persistence/__init__.py deleted file mode 100644 index f15edfb7d..000000000 --- a/plugins/python/parser/src/my_ast/persistence/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__all__ = ["build_action", "build_source_target", "file_dto", "file_content_dto", "persistence", "class_dto", - "function_dto", "import_dto.py", "variable_dto", "base_dto"] diff --git a/plugins/python/parser/src/my_ast/persistence/base_dto.py b/plugins/python/parser/src/my_ast/persistence/base_dto.py deleted file mode 100644 index 54a03e048..000000000 --- a/plugins/python/parser/src/my_ast/persistence/base_dto.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import List, Set - -from my_ast.common.file_position import FilePosition - - -class UsageDTO: - def __init__(self, file_position: FilePosition): - self.file_position: FilePosition = file_position - - -class DeclarationDTO: - def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], - usages: List[UsageDTO]): - self.name: str = name - self.qualified_name: str = qualified_name - self.file_position: FilePosition = file_position - self.type: Set[str] = types # qualified names - self.usages: List[UsageDTO] = usages diff --git a/plugins/python/parser/src/my_ast/persistence/build_action.py b/plugins/python/parser/src/my_ast/persistence/build_action.py deleted file mode 100644 index 55e3575ff..000000000 --- a/plugins/python/parser/src/my_ast/persistence/build_action.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import List - -import my_ast.persistence.build_source_target as bst - - -class BuildAction: - def __init__(self, command: str, sources: List[bst.BuildSource], targets: List[bst.BuildTarget]): - self.command = command - self.type = 0 # compile - self.build_sources: List[bst.BuildSource] = sources - self.build_target: List[bst.BuildTarget] = targets diff --git a/plugins/python/parser/src/my_ast/persistence/build_source_target.py b/plugins/python/parser/src/my_ast/persistence/build_source_target.py deleted file mode 100644 index 48cc2a131..000000000 --- a/plugins/python/parser/src/my_ast/persistence/build_source_target.py +++ /dev/null @@ -1,14 +0,0 @@ -import my_ast.persistence.build_action as ba -from my_ast.persistence.file_dto import FileDTO - - -class BuildSource: - def __init__(self, file: FileDTO, action: ba.BuildAction): - self.file: FileDTO = file - self.action: ba.BuildAction = action - - -class BuildTarget: - def __init__(self, file: FileDTO, action: ba.BuildAction): - self.file: FileDTO = file - self.action: ba.BuildAction = action diff --git a/plugins/python/parser/src/my_ast/persistence/class_dto.py b/plugins/python/parser/src/my_ast/persistence/class_dto.py deleted file mode 100644 index 67922ad1c..000000000 --- a/plugins/python/parser/src/my_ast/persistence/class_dto.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import Set, List - -from my_ast.common.file_position import FilePosition -from my_ast.persistence.base_dto import UsageDTO, DeclarationDTO -from my_ast.persistence.documented_dto import DocumentedDTO - - -class ClassMembersDTO: - def __init__(self): - self.methods: List[str] = [] - self.static_methods: List[str] = [] - self.attributes: List[str] = [] - self.static_attributes: List[str] = [] - self.classes: List[str] = [] - - -class ClassDeclarationDTO(DeclarationDTO, DocumentedDTO): - def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], - usages: List[UsageDTO], base_classes: Set[str], members: ClassMembersDTO, documentation: str): - DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) - DocumentedDTO.__init__(self, documentation) - self.base_classes: Set[str] = base_classes - self.members: ClassMembersDTO = members diff --git a/plugins/python/parser/src/my_ast/persistence/documented_dto.py b/plugins/python/parser/src/my_ast/persistence/documented_dto.py deleted file mode 100644 index 271e09b67..000000000 --- a/plugins/python/parser/src/my_ast/persistence/documented_dto.py +++ /dev/null @@ -1,3 +0,0 @@ -class DocumentedDTO: - def __init__(self, documentation: str): - self.documentation: str = documentation diff --git a/plugins/python/parser/src/my_ast/persistence/file.py b/plugins/python/parser/src/my_ast/persistence/file.py deleted file mode 100644 index f4d833539..000000000 --- a/plugins/python/parser/src/my_ast/persistence/file.py +++ /dev/null @@ -1,38 +0,0 @@ -from enum import unique, Enum -from pathlib import PurePath, Path -from my_ast.file_info import ProcessStatus, FileInfo -from my_ast.persistence.file_content import FileContent - - -@unique -class ParseStatus(Enum): - PSNONE = 0 - PSPARTIALLYPARSED = 1 - PSFULLYPARSED = 2 - - -class File: - def __init__(self, file_info: FileInfo): - # id? - self.type: str = 'Python' # dir, unknown, binary? - self.path: PurePath = file_info.path - self.file_name: str = file_info.file - self.timestamp = Path(file_info.path).stat().st_mtime # st_birthtime - self.content: FileContent = FileContent(self.get_content(file_info.path)) - self.parent: PurePath = file_info.path.parent - self.parse_status: ParseStatus = self.get_parse_status(file_info.status) - - @staticmethod - def get_content(file: PurePath) -> str: - with open(file) as f: - return f.read() - - @staticmethod - def get_parse_status(status: ProcessStatus) -> ParseStatus: - if status == ProcessStatus.WAITING: - return ParseStatus.PSNONE - elif status == ProcessStatus.PREPROCESSED: - return ParseStatus.PSPARTIALLYPARSED - elif status == ProcessStatus.PROCESSED: - return ParseStatus.PSFULLYPARSED - assert False diff --git a/plugins/python/parser/src/my_ast/persistence/file_content.py b/plugins/python/parser/src/my_ast/persistence/file_content.py deleted file mode 100644 index 2e7baf9ca..000000000 --- a/plugins/python/parser/src/my_ast/persistence/file_content.py +++ /dev/null @@ -1,3 +0,0 @@ -class FileContent: - def __init__(self, content: str): - self.content: str = content diff --git a/plugins/python/parser/src/my_ast/persistence/file_content_dto.py b/plugins/python/parser/src/my_ast/persistence/file_content_dto.py deleted file mode 100644 index 1487d9259..000000000 --- a/plugins/python/parser/src/my_ast/persistence/file_content_dto.py +++ /dev/null @@ -1,3 +0,0 @@ -class FileContentDTO: - def __init__(self, content: str): - self.content: str = content diff --git a/plugins/python/parser/src/my_ast/persistence/file_dto.py b/plugins/python/parser/src/my_ast/persistence/file_dto.py deleted file mode 100644 index d66192340..000000000 --- a/plugins/python/parser/src/my_ast/persistence/file_dto.py +++ /dev/null @@ -1,23 +0,0 @@ -from enum import unique, Enum - -from my_ast.persistence.documented_dto import DocumentedDTO -from my_ast.persistence.file_content_dto import FileContentDTO - - -@unique -class ParseStatus(Enum): - PSNONE = 0 - PSPARTIALLYPARSED = 1 - PSFULLYPARSED = 2 - - -class FileDTO(DocumentedDTO): - def __init__(self): - super().__init__("") - self.type: str = 'Python' # dir, unknown, binary? - self.path: str = "" - self.file_name: str = "" - self.timestamp = "" - self.content: FileContentDTO = FileContentDTO("") - self.parent: str = "" - self.parse_status: int = 0 # PSNONE diff --git a/plugins/python/parser/src/my_ast/persistence/function_dto.py b/plugins/python/parser/src/my_ast/persistence/function_dto.py deleted file mode 100644 index af7a450f9..000000000 --- a/plugins/python/parser/src/my_ast/persistence/function_dto.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Set, List - -from my_ast.common.file_position import FilePosition -from my_ast.persistence.base_dto import DeclarationDTO, UsageDTO -from my_ast.persistence.documented_dto import DocumentedDTO - - -class FunctionDeclarationDTO(DeclarationDTO, DocumentedDTO): - def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], - usages: List[UsageDTO], documentation: str): - DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) - DocumentedDTO.__init__(self, documentation) diff --git a/plugins/python/parser/src/my_ast/persistence/import_dto.py b/plugins/python/parser/src/my_ast/persistence/import_dto.py deleted file mode 100644 index c514fa4d4..000000000 --- a/plugins/python/parser/src/my_ast/persistence/import_dto.py +++ /dev/null @@ -1,17 +0,0 @@ -from pathlib import PurePath -from typing import Set, Dict - - -class ImportDTO: - def __init__(self, importer: PurePath): - self.importer: str = str(importer) - self.imported_modules: Set[str] = set() - self.imported_symbols: Dict[str, Set[str]] = {} - - def add_module_import(self, imported: PurePath): - self.imported_modules.add(str(imported)) - - def add_symbol_import(self, imported: PurePath, symbol_qualified_name: str): - if str(imported) not in self.imported_symbols: - self.imported_symbols[str(imported)] = set() - self.imported_symbols[str(imported)].add(symbol_qualified_name) diff --git a/plugins/python/parser/src/my_ast/persistence/persistence.py b/plugins/python/parser/src/my_ast/persistence/persistence.py deleted file mode 100644 index 5248d7937..000000000 --- a/plugins/python/parser/src/my_ast/persistence/persistence.py +++ /dev/null @@ -1,48 +0,0 @@ -from my_ast.persistence.file_dto import FileDTO -from my_ast.persistence.variable_dto import VariableDeclarationDTO -from my_ast.persistence.function_dto import FunctionDeclarationDTO -from my_ast.persistence.class_dto import ClassDeclarationDTO -from my_ast.persistence.import_dto import ImportDTO - - -class ModelPersistence: - def __init__(self, c_persistence): - self.c_persistence = c_persistence - self.check_c_persistence() - - def check_c_persistence(self): - if self.c_persistence is None: - return - assert hasattr(self.c_persistence, 'persist_file') - assert hasattr(self.c_persistence, 'persist_variable') - assert hasattr(self.c_persistence, 'persist_function') - assert hasattr(self.c_persistence, 'persist_class') - assert hasattr(self.c_persistence, 'persist_module_import') - - def persist_file(self, file: FileDTO) -> None: - if self.c_persistence is not None: - self.c_persistence.perist_file(file) - - def persist_variable(self, declaration: VariableDeclarationDTO) -> None: - if self.c_persistence is not None: - self.c_persistence.persist_variable(declaration) - - def persist_function(self, declaration: FunctionDeclarationDTO) -> None: - if self.c_persistence is not None: - self.c_persistence.persist_function(declaration) - - def persist_class(self, declaration: ClassDeclarationDTO) -> None: - if self.c_persistence is not None: - self.c_persistence.persist_class(declaration) - - def persist_import(self, imports: ImportDTO) -> None: - if self.c_persistence is not None: - self.c_persistence.persist_module_import(imports) - - -model_persistence = ModelPersistence(None) - - -def init_persistence(c_persistence): - global model_persistence - model_persistence = ModelPersistence(c_persistence) diff --git a/plugins/python/parser/src/my_ast/persistence/variable_dto.py b/plugins/python/parser/src/my_ast/persistence/variable_dto.py deleted file mode 100644 index 0e3b7bca5..000000000 --- a/plugins/python/parser/src/my_ast/persistence/variable_dto.py +++ /dev/null @@ -1,5 +0,0 @@ -from my_ast.persistence.base_dto import DeclarationDTO - - -class VariableDeclarationDTO(DeclarationDTO): - pass diff --git a/plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py b/plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py deleted file mode 100644 index 85b917a3d..000000000 --- a/plugins/python/parser/src/my_ast/placeholder_function_declaration_cache.py +++ /dev/null @@ -1,49 +0,0 @@ -import ast -from typing import Dict, Tuple, Optional, Set, Union, List - -from my_ast.common.hashable_list import OrderedHashableList -from my_ast.function_data import FunctionDeclaration -from my_ast.type_data import DeclarationType - - -# TODO: redefined functions? -class PlaceholderFunctionDeclarationCache: - def __init__(self): - self.cache: \ - Dict[FunctionDeclaration, - Tuple[Union[ast.FunctionDef, ast.AsyncFunctionDef], - Dict[OrderedHashableList[DeclarationType], Set[DeclarationType]]]] = {} - - def add_function_declaration(self, declaration: FunctionDeclaration, - function_def: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: - self.cache[declaration] = (function_def, {}) - - def get_function_def(self, declaration: FunctionDeclaration) \ - -> Optional[Union[ast.FunctionDef, ast.AsyncFunctionDef]]: - if declaration in self.cache: - return self.cache[declaration][0] - return None - - def get_function_return_type(self, declaration: FunctionDeclaration, - param_types: OrderedHashableList[DeclarationType]) -> Optional[Set[DeclarationType]]: - if declaration in self.cache and param_types in self.cache[declaration][1]: - return self.cache[declaration][1][param_types] - return None - - def get_functions_return_type(self, declarations: Set[FunctionDeclaration], - param_types: OrderedHashableList[DeclarationType]) \ - -> Set[DeclarationType]: - types = set() - for declaration in declarations: - t = self.get_function_return_type(declaration, param_types) - if t is not None: - types.update(t) - return types - - def add_function_return_type(self, declaration: FunctionDeclaration, - param_types: OrderedHashableList[DeclarationType], return_type: Set[DeclarationType]) \ - -> None: - if declaration in self.cache and param_types not in self.cache[declaration][1]: - self.cache[declaration][1][param_types] = return_type - elif declaration not in self.cache: - assert False, "Key not found" diff --git a/plugins/python/parser/src/my_ast/preprocessed_data.py b/plugins/python/parser/src/my_ast/preprocessed_data.py deleted file mode 100644 index f12ddd2b6..000000000 --- a/plugins/python/parser/src/my_ast/preprocessed_data.py +++ /dev/null @@ -1,8 +0,0 @@ -from my_ast.base_data import Declaration -from my_ast.common.file_position import FilePosition - - -class PreprocessedDeclaration(Declaration): - def __init__(self, name: str): - file_position = FilePosition.get_empty_file_position() - Declaration.__init__(self, name, "", file_position) # TODO: need qualified name? diff --git a/plugins/python/parser/src/my_ast/preprocessed_file.py b/plugins/python/parser/src/my_ast/preprocessed_file.py deleted file mode 100644 index 2219e7dbe..000000000 --- a/plugins/python/parser/src/my_ast/preprocessed_file.py +++ /dev/null @@ -1,59 +0,0 @@ -import ast -from typing import Set, Any - -from my_ast.class_preprocessor import PreprocessedClassCollector -from my_ast.import_preprocessor import ImportTable -from my_ast.preprocessed_function import PreprocessedFunction -from my_ast.preprocessed_variable import PreprocessedVariable - - -class PreprocessedFile(ast.NodeVisitor): - def __init__(self): - self.class_collector = PreprocessedClassCollector() - self.preprocessed_functions: Set[PreprocessedFunction] = set() - self.preprocessed_variables: Set[PreprocessedVariable] = set() - self.import_table = ImportTable() - self.depth = 0 - self.documentation: str = "" - - def visit_Module(self, node: ast.Module) -> Any: - documentation = ast.get_docstring(node) - if documentation is not None: - self.documentation = documentation - self.generic_visit(node) - - def visit_Import(self, node: ast.Import) -> Any: - is_local = False - if self.depth > 0: - is_local = True - self.import_table.append_import(node, is_local) - self.generic_visit(node) - - def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: - is_local = False - if self.depth > 0: - is_local = True - self.import_table.append_import_from(node, is_local) - self.generic_visit(node) - - def visit_ClassDef(self, node: ast.ClassDef) -> Any: - self.class_collector.append_class(node) - self.depth += 1 - self.generic_visit(node) - self.depth -= 1 - self.class_collector.class_processed() - - def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: - if self.depth == 0: - self.preprocessed_functions.add(PreprocessedFunction(node.name)) - self.depth += 1 - self.generic_visit(node) - self.depth -= 1 - - def visit_Assign(self, node: ast.Assign) -> Any: - if self.depth == 0: - for target in node.targets: - if isinstance(target, ast.Name): - self.preprocessed_variables.add(PreprocessedVariable(target.id)) - else: - print("...") diff --git a/plugins/python/parser/src/my_ast/preprocessed_function.py b/plugins/python/parser/src/my_ast/preprocessed_function.py deleted file mode 100644 index c0531a8a3..000000000 --- a/plugins/python/parser/src/my_ast/preprocessed_function.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import List - -from my_ast.base_data import Declaration -from my_ast.preprocessed_data import PreprocessedDeclaration - - -class PreprocessedFunction(PreprocessedDeclaration): - def __init__(self, name): - super().__init__(name) - self.type_usages: List[Declaration] = [] - - def __eq__(self, other): - return isinstance(other, PreprocessedFunction) and self.name == other.name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/my_ast/preprocessed_variable.py b/plugins/python/parser/src/my_ast/preprocessed_variable.py deleted file mode 100644 index a4ebd1ee4..000000000 --- a/plugins/python/parser/src/my_ast/preprocessed_variable.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import List - -from my_ast.base_data import Declaration -from my_ast.preprocessed_data import PreprocessedDeclaration - - -class PreprocessedVariable(PreprocessedDeclaration): - def __init__(self, name): - super().__init__(name) - self.type_usages: List[Declaration] = [] - - def __eq__(self, other): - return isinstance(other, PreprocessedVariable) and self.name == other.name - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/my_ast/python_parser.py b/plugins/python/parser/src/my_ast/python_parser.py deleted file mode 100644 index 0c3db8f6c..000000000 --- a/plugins/python/parser/src/my_ast/python_parser.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -from pathlib import PurePath - -from my_ast.parse_exception import ParseException -from my_ast.parser import Parser -from my_ast.persistence.persistence import init_persistence - - -def parse(source_root: str, persistence): - print(source_root) - - print(persistence.f()) - - init_persistence(persistence) - - def directory_exception(path: PurePath) -> bool: - directory = os.path.basename(os.path.normpath(str(path))) - return directory.startswith('.') or directory == 'venv' - - def file_exception(path: PurePath) -> bool: - return False - - exception = ParseException(directory_exception, file_exception) - p = Parser([source_root], exception) - p.parse() - pass - - -if __name__ == '__main__': - parse("dummy", None) # error! diff --git a/plugins/python/parser/src/my_ast/scope.py b/plugins/python/parser/src/my_ast/scope.py deleted file mode 100644 index 4014cb6ae..000000000 --- a/plugins/python/parser/src/my_ast/scope.py +++ /dev/null @@ -1,177 +0,0 @@ -from typing import List, Optional -from abc import abstractmethod, ABC -from uuid import uuid4 - -from my_ast.base_data import Declaration, Usage -from my_ast.class_data import ClassDeclaration -from my_ast.import_preprocessor import ImportTable -from my_ast.logger import logger -from my_ast.function_data import FunctionDeclaration -from my_ast.type_data import InitVariablePlaceholderType -from my_ast.variable_data import VariableDeclaration - - -class Scope(ABC): - def __init__(self, name: str): - self.name: str = name - self.variable_declarations: List[VariableDeclaration] = [] - self.function_declarations: List[FunctionDeclaration] = [] - self.class_declarations: List[ClassDeclaration] = [] - # TODO: should use only self.declarations, and filter the others? (slower, but less memory) - self.declarations: List[Declaration] = [] - - def append_variable(self, variable_declaration: VariableDeclaration) -> None: - self.variable_declarations.append(variable_declaration) - self.declarations.append(variable_declaration) - logger.debug('Var: ' + variable_declaration.name + ' (' + str(variable_declaration.file_position) + ') {' + - self.get_type() + ' - ' + self.name + '} ' + variable_declaration.get_type_repr()) - - def append_function(self, function_declaration: FunctionDeclaration) -> None: - self.function_declarations.append(function_declaration) - self.declarations.append(function_declaration) - logger.debug('Func: ' + function_declaration.name + ' (' + str(function_declaration.file_position) + ') {' + - self.get_type() + ' - ' + self.name + '} ' + function_declaration.get_type_repr()) - - def append_class(self, class_declaration: ClassDeclaration) -> None: - self.class_declarations.append(class_declaration) - self.declarations.append(class_declaration) - logger.debug('Class: ' + class_declaration.name + ' (' + str(class_declaration.file_position) + ') {' + - self.get_type() + ' - ' + self.name + '} ') - - def get_variable(self, usage: Usage) -> Optional[VariableDeclaration]: - return self.get_variable_by_name(usage.name) - - def get_variable_by_name(self, name: str) -> Optional[VariableDeclaration]: - variables = [x for x in self.variable_declarations if x.name == name] - # TODO: fix this assert - # assert len(variables) <= 1 - if len(variables) > 0: - return variables[0] - else: - return None - - def get_function(self, usage: Usage) -> Optional[FunctionDeclaration]: - return self.get_function_by_name(usage.name) - - def get_function_by_name(self, name: str) -> Optional[FunctionDeclaration]: - functions = [x for x in self.function_declarations if x.name == name] - if len(functions) > 0: - return functions[-1] - else: - return None - - def get_class_by_name(self, name: str) -> Optional[ClassDeclaration]: - classes = [x for x in self.class_declarations if x.name == name] - if len(classes) > 0: - return classes[-1] - else: - return None - - @abstractmethod - def get_type(self) -> str: - pass - - -class LifetimeScope(Scope, ABC): - """Symbols declared inside this scope are deleted after exiting from it.""" - - def __init__(self, name: str): - super().__init__(name) - - -class ImportScope(ABC): - """Imported modules are available during this scope.""" - - def __init__(self): - self.import_table: ImportTable = ImportTable() - - -class GlobalScope(LifetimeScope, ImportScope): - """The first scope, which lives on with its module.""" - - def __init__(self): - LifetimeScope.__init__(self, '') - ImportScope.__init__(self) - - def get_type(self) -> str: - return "global" - - -class FunctionScope(LifetimeScope, ImportScope): - """Scope of the functions and methods.""" - - def __init__(self, name: str): - LifetimeScope.__init__(self, name) - ImportScope.__init__(self) - - def get_type(self) -> str: - return "function" - - -class ClassScope(LifetimeScope, ImportScope): - """Scope of the classes.""" - - def __init__(self, name: str): - LifetimeScope.__init__(self, name) - ImportScope.__init__(self) - self.init_placeholders: List[InitVariablePlaceholderType] = [] - - def get_type(self) -> str: - return "class" - - -class PartialLifetimeScope(LifetimeScope, ABC): - """Not all of the symbols are deleted after existing from it.""" - pass - - -class ExceptionScope(PartialLifetimeScope): - """The exception variable is deleted after the 'except' scope, - but all the other symbols declared in it, are alive after it.""" - - def __init__(self): - super().__init__('exception') - - def get_type(self) -> str: - return 'exception' - - -class LambdaScope(LifetimeScope): - """Scope of lambdas.""" - - def __init__(self): - super().__init__('lambda_' + uuid4().hex) - - def get_type(self) -> str: - return "lambda" - - -class GeneratorScope(LifetimeScope): - """Scope of generators.""" - - def __init__(self): - super().__init__('generator_' + uuid4().hex) - - def get_type(self) -> str: - return "generator" - - -class ConditionalScope(Scope): - """Scope of if.""" - - def __init__(self): - super().__init__('if') - - def get_type(self) -> str: - return "conditional" - - -# TODO: need WhileScope and ForScope? -class LoopScope(Scope): - """Scope of for and while.""" - - def __init__(self): - super().__init__('loop') - - def get_type(self) -> str: - return "loop" diff --git a/plugins/python/parser/src/my_ast/scope_manager.py b/plugins/python/parser/src/my_ast/scope_manager.py deleted file mode 100644 index 9f4b3f342..000000000 --- a/plugins/python/parser/src/my_ast/scope_manager.py +++ /dev/null @@ -1,383 +0,0 @@ -import ast -from pathlib import PurePath, Path -from typing import List, Type, Iterator, Callable, Optional, Union - -from my_ast.base_data import Declaration, ImportedDeclaration -from my_ast.class_data import ClassDeclaration -from my_ast.function_data import FunctionDeclaration -from my_ast.import_finder import ImportFinder -from my_ast.import_preprocessor import ImportTable -from my_ast.member_access_collector import MemberAccessCollectorIterator, MemberAccessCollector -from my_ast.persistence.import_dto import ImportDTO -from my_ast.persistence.persistence import model_persistence -from my_ast.placeholder_function_declaration_cache import PlaceholderFunctionDeclarationCache -from my_ast.scope import Scope, ClassScope, GlobalScope, FunctionScope, LifetimeScope, PartialLifetimeScope, \ - ConditionalScope, LoopScope, ImportScope -from my_ast.type_data import PlaceholderType -from my_ast.variable_data import VariableDeclaration, ModuleVariableDeclaration - - -class ScopeManager: - def __init__(self, current_file: PurePath, import_finder: ImportFinder, scopes: Optional[List[Scope]] = None): - self.current_file = current_file - self.import_finder = import_finder - if scopes is None: - self.scopes: List[Scope] = [GlobalScope()] - else: - self.scopes = scopes - self.placeholder_function_declaration_cache = PlaceholderFunctionDeclarationCache() - - def is_current_scope_instance(self, t: Type) -> bool: - return isinstance(self.scopes[-1], t) - - # TODO: local functions - def is_current_scope_method(self) -> bool: - for i in range(len(self.scopes) - 1, -1, -1): - if isinstance(self.scopes[i], LifetimeScope) and not isinstance(self.scopes[i], PartialLifetimeScope): - return isinstance(self.scopes[i], FunctionScope) and isinstance(self.scopes[i - 1], ClassScope) - return False - - def is_current_scope_init(self) -> bool: - return self.is_current_scope_method() and self.get_current_lifetime_scope().name == '__init__' - - def get_current_lifetime_scope(self) -> LifetimeScope: - for scope in self.reverse(): - if isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): - return scope - assert False # there should be always a GlobalScope, which is LifetimeScope - - def reverse(self) -> Iterator[Scope]: - for scope in reversed(self.scopes): - yield scope - - def iterate(self) -> Iterator[Scope]: - for scope in self.scopes: - yield scope - - def with_scope(self, scope: Scope, action: Callable[[], None]) -> None: - self.scopes.append(scope) - action() - self.persist_current_scope() - del self.scopes[-1] - - def get_size(self) -> int: - return len(self.scopes) - - def get_current_scope(self) -> Scope: - return self.scopes[-1] - - def get_global_scope(self) -> GlobalScope: - assert len(self.scopes) > 0 - global_scope = self.scopes[0] - if not isinstance(global_scope, GlobalScope): - assert False - return global_scope - - def append_variable_to_current_scope(self, new_variable: VariableDeclaration) -> None: - self.get_current_lifetime_scope().append_variable(new_variable) - - def append_exception_to_current_scope(self, new_variable: VariableDeclaration) -> None: - self.get_current_partial_lifetime_scope().append_variable(new_variable) - - def append_function_to_current_scope(self, new_function: FunctionDeclaration) -> None: - self.get_current_lifetime_scope().append_function(new_function) - - def append_class_to_current_scope(self, new_class: ClassDeclaration) -> None: - self.get_current_lifetime_scope().append_class(new_class) - - def append_variable_to_current_class_scope(self, new_variable: VariableDeclaration) -> None: - self.get_current_class_scope().append_variable(new_variable) - - def append_function_to_current_class_scope(self, new_function: FunctionDeclaration) -> None: - self.get_current_class_scope().append_function(new_function) - - def append_class_to_current_class_scope(self, new_class: ClassDeclaration) -> None: - self.get_current_class_scope().append_class(new_class) - - def append_module_variable_to_current_scope(self, module_variable: ModuleVariableDeclaration) -> None: - def is_same_declaration(var: VariableDeclaration) -> bool: - return isinstance(var, ModuleVariableDeclaration) and \ - var.imported_module_location == module_variable.imported_module_location - scope = self.get_current_lifetime_scope() - if len([v for v in scope.variable_declarations if is_same_declaration(v)]) == 0: - scope.append_variable(module_variable) - - def get_current_partial_lifetime_scope(self) -> Optional[LifetimeScope]: - for scope in self.reverse(): - if isinstance(scope, PartialLifetimeScope): - return scope - return None - - def get_current_class_scope(self) -> Optional[ClassScope]: - for scope in self.reverse(): - if isinstance(scope, ClassScope): - return scope - return None - - def get_current_scope_name(self) -> str: - return self.scopes[-1].name - - def is_inside_conditional_scope(self) -> bool: - for scope in self.reverse(): - if isinstance(scope, (ConditionalScope, LoopScope)): - return True - elif isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): - return False - assert False # GlobalScope is LifetimeScope - - def is_declaration_in_current_scope(self, name: str) -> bool: - return self.is_declaration_in_scope(name, self.scopes[-1]) - - def is_declaration_in_current_class(self, name: str) -> bool: - class_scope = self.get_current_class_scope() - if class_scope is not None: - return self.is_declaration_in_scope(name, class_scope) - return False - - def get_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: - for scope in self.reverse(): - declaration = self.get_declaration_from_scope(name, scope) - if declaration is not None: - from my_ast.variable_data import TypeVariable - if isinstance(declaration, TypeVariable): - print("NEED FIX: TypeVariable") - return declaration - current_class_scope = self.get_current_class_scope() - if current_class_scope is not None: - for init_placeholder in current_class_scope.init_placeholders: - if name == init_placeholder.name: - return init_placeholder - return PlaceholderType(name) - - def get_function_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: - for scope in self.reverse(): - declaration = self.get_function_declaration_from_scope(name, scope) - if declaration is not None: - return declaration - return PlaceholderType(name) - - def get_declaration_from_member_access(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: - iter(iterator) - members = [next(iterator)] - # TODO: MethodVariable? - if isinstance(members[0], MemberAccessCollector.MethodData): - declaration = self.get_function_declaration(members[0].name) - else: - declaration = self.get_declaration(members[0].name) - if not isinstance(declaration, PlaceholderType): - return declaration - global_scope = self.get_global_scope() - assert isinstance(global_scope, GlobalScope) - try: - module_variable = self.get_imported_module(iterator) - if module_variable is not None: - declaration = module_variable - else: - declaration_from_other_module = self.get_declaration_from_other_module(iterator) - if declaration_from_other_module is not None: - declaration = declaration_from_other_module - except StopIteration: - assert False - return declaration - - # import a.b.c -> c.f() vs a.b.c.f() - def get_declaration_from_other_module(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: - for scope in self.reverse(): - if isinstance(scope, ImportScope): - for module in reversed(scope.import_table.modules): - iter(iterator) - # TODO: is this alias check enough? - if module.module.alias is None and len(module.path) >= len(iterator.mac.call_list): - continue - is_same = True - current = next(iterator) - name = current.name - iter(iterator) - if module.module.alias is not None: - is_same = module.module.alias == name - else: - for i in range(0, len(module.path)): - name = next(iterator).name - if module.path[i] != name: - is_same = False - break - if is_same: - name = next(iterator).name - if module.is_module_import(): - other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) - if other_scope_manager is not None: - if isinstance(current, MemberAccessCollector.MemberData): - return other_scope_manager.get_function_declaration(name) - else: - return other_scope_manager.get_declaration(name) - for imp in module.imported: - if (imp.alias is not None and imp.alias == name) \ - or imp.name == name or imp.is_all_imported(): - other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) - if other_scope_manager is not None: - if isinstance(current, MemberAccessCollector.MemberData): - return other_scope_manager.get_function_declaration(name) - else: - return other_scope_manager.get_declaration(name) - return None - - def get_imported_module(self, iterator: MemberAccessCollectorIterator) -> Optional[ModuleVariableDeclaration]: - imported_module = None - for scope in self.reverse(): - if isinstance(scope, ImportScope): - for module in reversed(scope.import_table.modules): - iter(iterator) - if len(module.path) != len(iterator.mac.call_list): - continue - for i in range(0, len(module.path)): - next(iterator) - module_path = [m.name for m in iterator.mac.call_list] - module_path.reverse() - if module.path == module_path: - imported_module = module - break - if imported_module is not None and isinstance(scope, Scope): - for var in reversed(scope.variable_declarations): - if isinstance(var, ModuleVariableDeclaration) and \ - var.imported_module_location == imported_module.location: - return var - return None - - def get_nonlocal_variable(self, name: str) -> Optional[VariableDeclaration]: - assert len(self.scopes) > 2 - for scope in self.scopes[-2:0:-1]: - var = scope.get_variable_by_name(name) - if var is not None: - return var - return None - - def get_global_variable(self, name: str) -> Optional[VariableDeclaration]: - return self.scopes[0].get_variable_by_name(name) - - @staticmethod - def get_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: - for declaration in reversed(scope.declarations): - if declaration.name == name: - return declaration - return None - - @staticmethod - def get_function_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: - for func in reversed(scope.function_declarations): - if func.name == name: - return func - return None - - @staticmethod - def is_declaration_in_scope(name: str, scope: Scope) -> bool: - for var in scope.declarations: - if var.name == name: - return True - return False - - def get_current_class_declaration(self) -> Optional[ClassDeclaration]: - current_class_scope_found = False - for scope in self.reverse(): - if current_class_scope_found: - return scope.class_declarations[-1] - current_class_scope_found = isinstance(scope, ClassScope) - return None - - def set_global_import_table(self, import_table: ImportTable): - global_scope = self.scopes[0] - if isinstance(global_scope, GlobalScope): - global_scope.import_table = import_table - - def append_import(self, node: ast.Import): - for scope in self.reverse(): - if isinstance(scope, ImportScope): - scope.import_table.append_import(node, isinstance(scope, GlobalScope)) - return - assert False - - def append_import_from(self, node: ast.ImportFrom): - for scope in self.reverse(): - if isinstance(scope, ImportScope): - scope.import_table.append_import_from(node, isinstance(scope, GlobalScope)) - return - assert False - - def handle_import_scope(self): - if isinstance(self.scopes[-1], ImportScope): - for declaration in self.scopes[-1].declarations: - if isinstance(declaration, ImportedDeclaration): - declaration.imported_declaration.usages.extend(declaration.usages) - - def persist_current_scope(self): - # for declaration in self.scopes[-1].declarations: - # print(self.get_qualified_name_from_current_scope(declaration)) - - self.handle_import_scope() - - import_dto = ImportDTO(self.current_file) - - for var in self.scopes[-1].variable_declarations: - if isinstance(var, ImportedDeclaration): - import_dto.add_symbol_import(var.imported_declaration.file_position.file, - var.imported_declaration.qualified_name) - elif isinstance(var, ModuleVariableDeclaration): - import_dto.add_module_import(var.imported_module_location) - else: - model_persistence.persist_variable(var.create_dto()) - - for func in self.scopes[-1].function_declarations: - if isinstance(func, ImportedDeclaration): - import_dto.add_symbol_import(func.imported_declaration.file_position.file, - func.imported_declaration.qualified_name) - else: - model_persistence.persist_function(func.create_dto()) - - for cl in self.scopes[-1].class_declarations: - if isinstance(cl, ImportedDeclaration): - import_dto.add_symbol_import(cl.imported_declaration.file_position.file, - cl.imported_declaration.qualified_name) - else: - model_persistence.persist_class(cl.create_dto()) - - if len(import_dto.imported_modules) > 0 or len(import_dto.imported_symbols) > 0: - model_persistence.persist_import(import_dto) - - def get_qualified_name_from_current_scope(self, declaration_name: str) -> str: - qualified_name_parts = [declaration_name] - for scope in reversed(self.scopes): - if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope - qualified_name_parts.append(scope.name) - file_name = self.current_file.name - assert file_name[-3:] == '.py' - qualified_name_parts.append(file_name[:-3]) - directory = self.current_file.parent - while True: - init_file_location = Path(directory).joinpath("__init__.py") - if init_file_location.exists(): - qualified_name_parts.append(directory.name) - directory = directory.parent - else: - break - - return '.'.join(reversed(qualified_name_parts)) - - # @staticmethod - # def is_variable_declaration_in_scope(name: str, scope: Scope) -> bool: - # for var in scope.declarations: - # if var.name == name: - # return isinstance(var, VariableDeclaration) - # return False - # - # @staticmethod - # def is_function_declaration_in_scope(name: str, scope: Scope) -> bool: - # for var in scope.declarations: - # if var.name == name: - # return isinstance(var, FunctionDeclaration) - # return False - # - # @staticmethod - # def is_class_declaration_in_scope(name: str, scope: Scope) -> bool: - # for var in scope.declarations: - # if var.name == name: - # return isinstance(var, ClassDeclaration) - # return False diff --git a/plugins/python/parser/src/my_ast/symbol_collector.py b/plugins/python/parser/src/my_ast/symbol_collector.py deleted file mode 100644 index a6784222e..000000000 --- a/plugins/python/parser/src/my_ast/symbol_collector.py +++ /dev/null @@ -1,881 +0,0 @@ -import ast -from pathlib import PurePath -from typing import List, Optional, Any, Union, Set, TypeVar - -from my_ast.built_in_functions import get_built_in_function -from my_ast.common.file_position import FilePosition -from my_ast.common.utils import create_range_from_ast_node -from my_ast.function_symbol_collector_factory import FunctionSymbolCollectorFactory -from my_ast.symbol_collector_interface import SymbolCollectorBase -from my_ast.base_data import Declaration, Usage -from my_ast.class_data import ClassDeclaration, ImportedClassDeclaration -from my_ast.class_init_declaration import ClassInitDeclaration -from my_ast.class_preprocessor import PreprocessedClass -from my_ast.common.parser_tree import ParserTree -from my_ast.common.position import Range -from my_ast.import_finder import ImportFinder -from my_ast.import_preprocessor import ImportTable -from my_ast.logger import logger -from my_ast.member_access_collector import MemberAccessCollector -from my_ast.preprocessed_file import PreprocessedFile -from my_ast.preprocessed_function import PreprocessedFunction -from my_ast.preprocessed_variable import PreprocessedVariable -from my_ast.scope import Scope, ClassScope, FunctionScope, GeneratorScope, LambdaScope, \ - ExceptionScope, ConditionalScope, LoopScope, GlobalScope -from my_ast.function_data import FunctionDeclaration, ImportedFunctionDeclaration, \ - StaticFunctionDeclaration -from my_ast.scope_manager import ScopeManager -from my_ast.symbol_finder import SymbolFinder -from my_ast.type_data import PlaceholderType, InitVariablePlaceholderType -from my_ast.type_deduction import TypeDeduction, VariablePlaceholderType -from my_ast.variable_data import VariableDeclaration, ImportedVariableDeclaration, \ - ModuleVariableDeclaration, StaticVariableDeclaration, NonlocalVariableDeclaration, GlobalVariableDeclaration, \ - FunctionVariable, TypeVariable - - -class SymbolCollector(ast.NodeVisitor, SymbolFinder, SymbolCollectorBase): - def __init__(self, tree: ParserTree, current_file: PurePath, preprocessed_file: PreprocessedFile, - import_finder: ImportFinder, function_symbol_collector_factory: FunctionSymbolCollectorFactory): - SymbolCollectorBase.__init__(self, preprocessed_file, import_finder) - self.tree = tree - self.current_file = current_file - self.scope_manager: ScopeManager = ScopeManager(current_file, import_finder) - self.scope_manager.set_global_import_table(preprocessed_file.import_table) - self.current_function_declaration: List[FunctionDeclaration] = [] - self.function_symbol_collector_factory = function_symbol_collector_factory - self.type_deduction = TypeDeduction(self, self.scope_manager, self.function_symbol_collector_factory) - - def collect_symbols(self): - self.visit(self.tree.root.node) - self.post_process() - - def visit_Import(self, node: ast.Import) -> Any: - modules = ImportTable.convert_ast_import_to_import(node) - for module in modules: - if not module.is_module_import(): - continue - # scope.imports.append((module, self.import_finder.get_global_scope_by_location(module.location))) - self.scope_manager.append_module_variable_to_current_scope( - ModuleVariableDeclaration(module.path[-1], module.location, - FilePosition(self.current_file, module.range), - self.import_finder.get_global_scope_by_location(module.location))) - if len(self.scope_manager.scopes) > 1: - self.scope_manager.append_import(node) - self.generic_visit(node) - - def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: - modules = ImportTable.convert_ast_import_from_to_import(node) - for module in modules: - self.scope_manager.append_module_variable_to_current_scope( - ModuleVariableDeclaration(module.path[-1], module.location, - FilePosition(self.current_file, module.range), - self.import_finder.get_global_scope_by_location(module.location))) - if not module.is_module_import(): - scope_manager: ScopeManager = self.import_finder.get_scope_manager_by_location(module.location) - if scope_manager is None: - # import from library - continue - # in case of circular import this case is not valid (runtime error) -> must find declaration - file_position = FilePosition(self.current_file, Range.get_empty_range()) - for imported in module.imported: - if imported.is_all_imported(): - for declaration in scope_manager.get_global_scope().declarations: - if isinstance(declaration, VariableDeclaration): - var = ImportedVariableDeclaration( - declaration.name, file_position, declaration) - self.scope_manager.append_variable_to_current_scope(var) - self.imported_declaration_scope_map[var] = scope_manager - elif isinstance(declaration, FunctionDeclaration): - func = ImportedFunctionDeclaration( - declaration.name, file_position, declaration) - self.scope_manager.append_function_to_current_scope(func) - self.imported_declaration_scope_map[func] = scope_manager - elif isinstance(declaration, ClassDeclaration): - c = ImportedClassDeclaration( - declaration.name, file_position, declaration) - self.scope_manager.append_class_to_current_scope(c) - self.imported_declaration_scope_map[c] = scope_manager - else: - assert False - else: - # TODO: redefinition? - declaration = scope_manager.get_function_declaration(imported.name) - if isinstance(declaration, PlaceholderType): - declaration = scope_manager.get_declaration(imported.name) - if isinstance(declaration, PlaceholderType): - continue - elif isinstance(declaration, VariableDeclaration): - var = ImportedVariableDeclaration( - imported.name if imported.alias is None else imported.alias, - file_position, declaration) - self.scope_manager.append_variable_to_current_scope(var) - self.imported_declaration_scope_map[var] = scope_manager - elif isinstance(declaration, FunctionDeclaration): - func = ImportedFunctionDeclaration( - imported.name if imported.alias is None else imported.alias, - file_position, declaration) - self.scope_manager.append_function_to_current_scope(func) - self.imported_declaration_scope_map[func] = scope_manager - elif isinstance(declaration, ClassDeclaration): - c = ImportedClassDeclaration( - imported.name if imported.alias is None else imported.alias, - file_position, declaration) - self.scope_manager.append_class_to_current_scope(c) - self.imported_declaration_scope_map[c] = scope_manager - else: - assert False - if len(self.scope_manager.scopes) > 1: - self.scope_manager.append_import_from(node) - self.generic_visit(node) - - def visit_ClassDef(self, node: ast.ClassDef) -> Any: - base_classes = self.collect_base_classes(node) - r = create_range_from_ast_node(node) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name) - new_class = ClassDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), - base_classes, self.get_documentation(node)) - self.class_def_process_derived_members(new_class, base_classes) - # TODO: handle preprocessed classes - self.scope_manager.get_current_scope().append_class(new_class) - # TODO: check if has at least 1 parameter? (self) - # TODO: range could be the __init__ - self.scope_manager.append_function_to_current_scope( - ClassInitDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), [], new_class)) - self.scope_manager.append_variable_to_current_scope( - TypeVariable(node.name, qualified_name, FilePosition(self.current_file, r), new_class)) - - def action(): - self.generic_visit(node) - scope = self.scope_manager.get_current_scope() - assert isinstance(scope, ClassScope) - for var in scope.variable_declarations: - if isinstance(var, StaticVariableDeclaration) and var not in new_class.static_attributes: - new_class.static_attributes.append(var) - elif isinstance(var, VariableDeclaration) and var not in new_class.attributes: - new_class.attributes.append(var) - for func in scope.function_declarations: - if isinstance(func, StaticFunctionDeclaration): - new_class.static_methods.append(func) - elif isinstance(func, FunctionDeclaration): - new_class.methods.append(func) - else: - assert False - for c in scope.class_declarations: - new_class.classes.append(c) - - self.scope_manager.with_scope(ClassScope(node.name), action) - self.class_def_post_process(new_class) - - def collect_base_classes(self, node: ast.ClassDef) -> List[ClassDeclaration]: - base_classes = [] - for base_class in node.bases: - # declaration = None - base_class_types = self.type_deduction.get_current_type( - self.type_deduction.deduct_type(base_class, self.preprocessed_file)) - # if isinstance(base_class, ast.Name): - # declaration = self.scope_manager.get_declaration(base_class.id) - # elif isinstance(base_class, ast.Subscript): - # pass - # # declaration = self.scope_manager.get_declaration(base_class.value.id) - # elif isinstance(base_class, ast.Attribute): - # pass - # else: - # assert False - assert len(base_class_types) <= 1 - for base_class_type in base_class_types: - if isinstance(base_class_type, ClassDeclaration): - base_classes.append(base_class_type) - elif isinstance(base_class_type, TypeVariable): - base_classes.append(base_class_type.reference) - else: - print('') - return base_classes - - def class_def_process_derived_members(self, cl: ClassDeclaration, base_classes: List[ClassDeclaration]): - declaration_type = TypeVar('declaration_type', bound=Declaration) - - def collect_derived_members(members: List[List[declaration_type]]) -> List[declaration_type]: - declarations = [] - for base_class_members in members: - for member in base_class_members: - if all([m.name != member.name for m in declarations]): - declarations.append(member) - return declarations - - cl.attributes.extend(collect_derived_members([d.attributes for d in base_classes])) - cl.static_attributes.extend(collect_derived_members([d.static_attributes for d in base_classes])) - cl.methods.extend(collect_derived_members([d.methods for d in base_classes])) - cl.static_methods.extend(collect_derived_members([d.static_methods for d in base_classes])) - cl.classes.extend(collect_derived_members([d.classes for d in base_classes])) - - def class_def_post_process(self, cl: ClassDeclaration): - def remove_hidden_declarations(declarations: List[Declaration]): - for declaration in reversed(declarations): - hidden_declarations = [d for d in declarations if d.name == declaration.name and d is not declaration] - for hidden_declaration in hidden_declarations: - declarations.remove(hidden_declaration) - - remove_hidden_declarations(cl.attributes) - remove_hidden_declarations(cl.static_attributes) - remove_hidden_declarations(cl.methods) # TODO: singledispatchmethod - remove_hidden_declarations(cl.static_methods) - remove_hidden_declarations(cl.classes) - - def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: - self.visit_common_function_def(node) - - def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: - self.visit_common_function_def(node) - - def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: - r = create_range_from_ast_node(node) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name) - if self.is_static_method(node): - new_function = StaticFunctionDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), - [], self.get_documentation(node)) - else: - new_function = FunctionDeclaration(node.name, qualified_name, - FilePosition(self.current_file, r), [], self.get_documentation(node)) - self.scope_manager.append_function_to_current_scope(new_function) - new_func_var = FunctionVariable(node.name, qualified_name, FilePosition(self.current_file, r), new_function) - self.scope_manager.append_variable_to_current_scope(new_func_var) - self.current_function_declaration.append(new_function) - - # if node.name == '__init__' and isinstance(self.scopes[-1], ClassScope): # ctor ~ __init__ - - def action(): - # self.collect_parameters(node) - self.process_function_def(node) - if any(isinstance(x, PlaceholderType) for x in self.current_function_declaration[-1].type): - self.scope_manager.placeholder_function_declaration_cache. \ - add_function_declaration(new_function, node) - - self.scope_manager.with_scope(FunctionScope(node.name), action) - del self.current_function_declaration[-1] - - def get_documentation(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef]) -> str: - documentation = ast.get_docstring(node) - if documentation is None: - return "" - else: - return documentation - - def collect_parameters(self, node: ast.FunctionDef) -> None: - for param in node.args.args: - if hasattr(param, 'arg'): # ast.Name - r = create_range_from_ast_node(node) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(param.arg) - new_variable = VariableDeclaration(param.arg, qualified_name, FilePosition(self.current_file, r)) - # TODO: first parameter could be anything, not just self + static method check! - if param.arg == 'self' and self.scope_manager.is_current_scope_method() and node.args.args[0] is param: - new_variable.type.add(self.scope_manager.get_current_class_declaration()) - elif node.name == '__init__': - init_var_placeholder_type = InitVariablePlaceholderType(param.arg) - new_variable.type.add(init_var_placeholder_type) - self.scope_manager.get_current_class_scope().init_placeholders.append(init_var_placeholder_type) - else: - new_variable.type.add(VariablePlaceholderType(param.arg)) - self.scope_manager.append_variable_to_current_scope(new_variable) - else: - assert False, "Parameter with unknown variable name" - - def process_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: - def get_first_param() -> Optional[ast.arg]: - if len(node.args.posonlyargs) > 0: - return node.args.posonlyargs[0] - if len(node.args.args) > 0: - return node.args.args[0] - if node.args.vararg is not None: - return node.args.vararg - if len(node.args.kwonlyargs) > 0: - return node.args.kwonlyargs[0] - if node.args.kwarg is not None: - return node.args.kwarg - return None - - def process_arg(param: Optional[ast.arg]): - if param is None: - return - r = create_range_from_ast_node(arg) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(param.arg) - new_variable = VariableDeclaration(param.arg, qualified_name, FilePosition(self.current_file, r)) - if param.arg == 'self' and is_method and get_first_param() is param: - new_variable.type.add(self.scope_manager.get_current_class_declaration()) - elif is_init: - init_var_placeholder_type = InitVariablePlaceholderType(param.arg) - new_variable.type.add(init_var_placeholder_type) - self.scope_manager.get_current_class_scope().init_placeholders.append(init_var_placeholder_type) - else: - new_variable.type.add(VariablePlaceholderType(param.arg)) - self.scope_manager.append_variable_to_current_scope(new_variable) - - is_init = self.scope_manager.is_current_scope_init() - is_method = self.scope_manager.is_current_scope_method() - for arg in node.args.args: # simple arguments - process_arg(arg) - process_arg(node.args.vararg) # *arg - for arg in node.args.kwonlyargs: # arguments between vararg and kwarg - process_arg(arg) - process_arg(node.args.kwarg) # **arg - for default in node.args.defaults: # last N args default value - self.visit(default) - for default in node.args.kw_defaults: # default values of kwonlyargs (all) - self.visit(default) - for arg in node.args.posonlyargs: # arguments before / "parameter" (after -> args) - process_arg(arg) - - for decorator in node.decorator_list: - self.visit(decorator) - - if node.returns is not None: - self.visit(node.returns) - - for stmt in node.body: - self.visit(stmt) - - def visit_Return(self, node: ast.Return) -> Any: - assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) - if node.value is not None: - types = self.type_deduction.deduct_type(node, self.preprocessed_file) - if len(types) == 0: - self.current_function_declaration[-1].type.add(VariablePlaceholderType('')) - else: - self.current_function_declaration[-1].type.update(types) - self.generic_visit(node) - - def visit_Yield(self, node: ast.Yield) -> Any: - assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) - # TODO: ~Return - self.current_function_declaration[-1].type.update( - self.type_deduction.deduct_type(node, self.preprocessed_file)) - self.generic_visit(node) - - def visit_YieldFrom(self, node: ast.YieldFrom) -> Any: - assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) - # TODO: ~Return - self.current_function_declaration[-1].type.update( - self.type_deduction.deduct_type(node, self.preprocessed_file)) - self.generic_visit(node) - - def visit_NamedExpr(self, node: ast.NamedExpr) -> Any: - fh = None - if isinstance(node.target, ast.Attribute): - fh = MemberAccessCollector(node.target) - name = fh.call_list[0].name - elif isinstance(node.target, ast.Name): - name = node.target.id - elif isinstance(node.target, (ast.Subscript, ast.Starred, ast.List, ast.Tuple)): - return - else: - assert False - - r = create_range_from_ast_node(node.target) - types = self.type_deduction.deduct_type(node.value, self.preprocessed_file) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) - if self.is_static_variable(node.target): - new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), - self.type_deduction.get_current_type(types)) - else: - new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), - self.type_deduction.get_current_type(types)) - if self.scope_manager.is_current_scope_init() and fh is not None and len(fh.call_list) == 2 and \ - fh.call_list[-1].name == 'self': - if not self.scope_manager.is_declaration_in_current_class(new_variable.name): - self.scope_manager.append_variable_to_current_class_scope(new_variable) - self.scope_manager.get_current_class_declaration().attributes.append(new_variable) - elif isinstance(node.target, ast.Name): - if self.scope_manager.is_inside_conditional_scope() and \ - not isinstance(declaration := self.scope_manager.get_declaration(name), PlaceholderType): - declaration.type.update(new_variable.type) - else: - self.scope_manager.append_variable_to_current_scope(new_variable) - else: - print('') - self.generic_visit(node) - - def visit_Assign(self, node: ast.Assign) -> Any: - for target in node.targets: - mac = None - if isinstance(target, ast.Attribute): - mac = MemberAccessCollector(target) - name = mac.call_list[0].name - elif isinstance(target, ast.Name): - name = target.id - elif isinstance(target, (ast.Subscript, ast.Starred, ast.List, ast.Tuple)): - return - else: - assert False - - r = create_range_from_ast_node(target) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) - types = self.type_deduction.get_current_type( - self.type_deduction.deduct_type(node.value, self.preprocessed_file)) - if self.is_static_variable(target): - new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), - self.type_deduction.get_current_type(types)) - else: - new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), - self.type_deduction.get_current_type(types)) - if self.scope_manager.is_current_scope_init() and mac is not None and len(mac.call_list) == 2 and \ - mac.call_list[-1].name == 'self': - if not self.scope_manager.is_declaration_in_current_class(new_variable.name): - self.scope_manager.append_variable_to_current_class_scope(new_variable) - self.scope_manager.get_current_class_declaration().attributes.append(new_variable) - elif isinstance(target, ast.Name): - if self.scope_manager.is_inside_conditional_scope() and \ - not isinstance(declaration := self.scope_manager.get_declaration(name), PlaceholderType): - declaration.type.update(new_variable.type) - else: - self.scope_manager.append_variable_to_current_scope(new_variable) - else: - print('') - - self.generic_visit(node) - - def visit_AnnAssign(self, node: ast.AnnAssign) -> Any: - fh = None - if isinstance(node.target, ast.Attribute): - fh = MemberAccessCollector(node.target) - name = fh.call_list[0].name - elif isinstance(node.target, ast.Name): - name = node.target.id - elif isinstance(node.target, (ast.Subscript, ast.Starred, ast.List, ast.Tuple)): - return - else: - assert False - - r = create_range_from_ast_node(node.target) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) - types = self.type_deduction.get_current_type( - self.type_deduction.deduct_type(node.value, self.preprocessed_file)) - if self.is_static_variable(node.target): - new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), types) - else: - new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), types) - if self.scope_manager.is_current_scope_init() and fh is not None and \ - len(fh.call_list) == 2 and fh.call_list[-1].name == 'self': - if not self.scope_manager.is_declaration_in_current_class(new_variable.name): - self.scope_manager.append_variable_to_current_class_scope(new_variable) # TODO: use type of self - elif isinstance(node.target, ast.Name): - if not self.scope_manager.is_declaration_in_current_scope(new_variable.name): - self.scope_manager.append_variable_to_current_scope(new_variable) # TODO: not only in last scope - - def visit_For(self, node: ast.For) -> Any: - self.visit_common_for(node) - - def visit_AsyncFor(self, node: ast.AsyncFor) -> Any: - self.visit_common_for(node) - - def visit_common_for(self, node: Union[ast.For, ast.AsyncFor]): - def action(): - if not isinstance(node.target, ast.Tuple) and \ - not self.scope_manager.is_declaration_in_current_scope(node.target.id): - r = create_range_from_ast_node(node.target) - types = self.type_deduction.deduct_type(node.iter, self.preprocessed_file) - qn = self.scope_manager.get_qualified_name_from_current_scope(node.target.id) - new_variable = VariableDeclaration(node.target.id, qn, FilePosition(self.current_file, r), types) - self.scope_manager.append_variable_to_current_scope(new_variable) - elif isinstance(node.target, ast.Tuple): - for var in node.target.elts: - r = create_range_from_ast_node(var) - # TODO: type - types = set() - # self.type_deduction.deduct_type(node.iter, self.preprocessed_file.class_collector) - qn = self.scope_manager.get_qualified_name_from_current_scope(var.id) - new_variable = VariableDeclaration(var.id, qn, FilePosition(self.current_file, r), types) - self.scope_manager.append_variable_to_current_scope(new_variable) - else: - # types = self.type_deduction.deduct_type(...) - # get variable - # add types - pass - self.generic_visit(node) - - self.scope_manager.with_scope(LoopScope(), action) - - def visit_With(self, node: ast.With) -> Any: - self.common_with_visit(node) - - def visit_AsyncWith(self, node: ast.AsyncWith) -> Any: - self.common_with_visit(node) - - def common_with_visit(self, node: Union[ast.With, ast.AsyncWith]) -> Any: - def action(): - for with_item in node.items: - if with_item.optional_vars is not None: - r = create_range_from_ast_node(with_item.optional_vars) - qn = self.scope_manager.get_qualified_name_from_current_scope(with_item.optional_vars.id) - types = self.type_deduction.deduct_type(with_item.context_expr, self.preprocessed_file) - new_variable = VariableDeclaration(with_item.optional_vars.id, qn, - FilePosition(self.current_file, r), types) - self.scope_manager.append_variable_to_current_scope(new_variable) - self.generic_visit(node) - - self.scope_manager.with_scope(LoopScope(), action) - - def visit_ExceptHandler(self, node: ast.ExceptHandler) -> Any: - def action(): - if node.name is not None: - r = create_range_from_ast_node(node) - types = self.type_deduction.deduct_type(node.type, self.preprocessed_file) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name) - new_variable = VariableDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), types) - self.scope_manager.append_exception_to_current_scope(new_variable) - self.generic_visit(node) - - self.scope_manager.with_scope(ExceptionScope(), action) - - def visit_Global(self, node: ast.Global) -> Any: - r = create_range_from_ast_node(node) - for name in node.names: - reference = self.scope_manager.get_global_variable(name) - if reference is None: - continue - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) - self.scope_manager.append_variable_to_current_scope( - GlobalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) - if reference is not None: - reference.usages.append(Usage(name, FilePosition(self.current_file, r))) - - def visit_GeneratorExp(self, node: ast.GeneratorExp) -> Any: - self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) - - def visit_ListComp(self, node: ast.ListComp) -> Any: - self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) - - def visit_SetComp(self, node: ast.SetComp) -> Any: - self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) - - def visit_DictComp(self, node: ast.DictComp) -> Any: - self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) - - def visit_Lambda(self, node: ast.Lambda) -> Any: - def action(): - for var in node.args.args: - r = create_range_from_ast_node(var) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(var.arg) - new_variable = VariableDeclaration(var.arg, qualified_name, FilePosition(self.current_file, r)) - self.scope_manager.append_variable_to_current_scope(new_variable) - self.generic_visit(node) - - self.scope_manager.with_scope(LambdaScope(), action) - - def visit_comprehension(self, node: ast.comprehension) -> Any: - if isinstance(node.target, ast.Name): - r = create_range_from_ast_node(node.target) - qn = self.scope_manager.get_qualified_name_from_current_scope(node.target.id) - types = self.type_deduction.deduct_type(node.iter, self.preprocessed_file) - new_variable = VariableDeclaration(node.target.id, qn, FilePosition(self.current_file, r), types) - self.scope_manager.append_variable_to_current_scope(new_variable) - elif isinstance(node.target, ast.Tuple): - for var in node.target.elts: - r = create_range_from_ast_node(var) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(var.id) - types = self.type_deduction.deduct_type(node.iter, self.preprocessed_file) - new_variable = VariableDeclaration(var.id, qualified_name, FilePosition(self.current_file, r), types) - self.scope_manager.append_variable_to_current_scope(new_variable) - else: - assert False, 'Comprehension target type not handled: ' + type(node.target) - self.generic_visit(node) - - def visit_Nonlocal(self, node: ast.Nonlocal) -> Any: - r = create_range_from_ast_node(node) - for name in node.names: - reference = self.scope_manager.get_nonlocal_variable(name) - if reference is None: - continue - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name) - self.scope_manager.append_variable_to_current_scope( - NonlocalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) - if reference is not None: - reference.usages.append(Usage(name, FilePosition(self.current_file, r))) - - def visit_Name(self, node: ast.Name) -> Any: - if isinstance(node.ctx, ast.Store): - var = self.get_declaration(node) - if var is not None: - other = self.create_var_usage(node.id, node) - if not list(var)[0].is_same_usage(other): - self.append_variable_usage(node.id, node) - else: - r = create_range_from_ast_node(node) - qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.id) - self.scope_manager.append_variable_to_current_scope( - VariableDeclaration(node.id, qualified_name, FilePosition(self.current_file, r))) - assert False, 'Assignment target, was not handled at ' + str(r) - elif isinstance(node.ctx, (ast.Load, ast.Del)): - self.append_variable_usage(node.id, node) - else: - assert False, "Unknown context" - self.generic_visit(node) - - def visit_Attribute(self, node: ast.Attribute) -> Any: - self.append_variable_usage(node.attr, node) - self.generic_visit(node) - - def visit_Call(self, node: ast.Call) -> Any: - types = self.type_deduction.deduct_type(node, self.preprocessed_file) - if len(types) == 0: - print('ph') - for t in types: - from my_ast.type_data import PlaceholderType - if isinstance(t, PlaceholderType): - print('ph') - for t in self.type_deduction.get_current_type(types): - from my_ast.type_data import PlaceholderType - if isinstance(t, PlaceholderType): - print('ph') - - self.generic_visit(node.func) - - for arg in node.args: - self.visit(arg) - for keyword in node.keywords: - self.visit(keyword.value) - - self.append_function_usage(node) - - def visit_Delete(self, node: ast.Delete) -> Any: - # TODO: remove variable (if not -> only runtime error) - self.generic_visit(node) - - def visit_If(self, node: ast.If) -> Any: - # TODO: can be recursive (eg. if..elif..elif..else..), is it a problem? - self.scope_manager.with_scope(ConditionalScope(), lambda: self.generic_visit(node)) - - @staticmethod - def get_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: - for var in scope.declarations: - if var.name == name: - return var - return None - - @staticmethod - def is_declaration_in_scope(name: str, scope: Scope) -> bool: - for var in scope.declarations: - if var.name == name: - return True - return False - - @staticmethod - def is_variable_declaration_in_scope(name: str, scope: Scope) -> bool: - for var in scope.declarations: - if var.name == name: - return isinstance(var, VariableDeclaration) - return False - - @staticmethod - def is_function_declaration_in_scope(name: str, scope: Scope) -> bool: - for var in scope.declarations: - if var.name == name: - return isinstance(var, FunctionDeclaration) - return False - - @staticmethod - def is_class_declaration_in_scope(name: str, scope: Scope) -> bool: - for var in scope.declarations: - if var.name == name: - return isinstance(var, ClassDeclaration) - return False - - def append_variable_usage(self, name: str, node: (ast.Name, ast.Attribute, ast.Subscript)) -> None: - usage = self.create_var_usage(name, node) - variable_declarations = self.get_declaration(node) - for var in variable_declarations: - if var is not None and not isinstance(var, PlaceholderType): - var.usages.append(usage) - logger.debug('Var Usage: ' + str(usage) + ' ' + var.get_type_repr()) - return - # TODO: annotations -> assert - assert True, "Variable not found: " + usage.name + " (usage: line - " + str(usage.file_position) + ")" - - def append_function_usage(self, node: ast.Call) -> None: - usage = self.create_function_usage(node) - mac = self.member_access_collector_type(node) - is_method_call = (len(mac.call_list) == 2 and - isinstance(mac.call_list[1], MemberAccessCollector.AttributeData) and - mac.call_list[1].name == 'self') - if not (len(mac.call_list) == 1 or is_method_call): - # functions from another module and modules - # return # TODO: handle these cases (type deduction) - pass - function_declarations = self.get_declaration(node) - for func in function_declarations: - if func is not None and not isinstance(func, PlaceholderType): - func.usages.append(usage) - logger.debug('Func Usage: ' + str(usage) + ' ' + func.get_type_repr()) - return - if self.is_builtin(usage.name): - self.append_builtin_usage(usage) - return - # TODO: imported functions/methods from libraries not in the project (builtins!) - assert True, "Function not found: " + usage.name + " (usage: start position - " + str(usage.file_position) + ")" - - def is_builtin(self, name: str) -> bool: - return get_built_in_function(name) is not None - - def append_builtin_usage(self, usage: Usage): - bif = get_built_in_function(usage.name) - if bif is not None: - bif.usages.append(usage) - - def get_declaration(self, node: (ast.Name, ast.Attribute, ast.Call, ast.Subscript)) -> Optional[Set[Declaration]]: - if isinstance(node, ast.Name): - declaration = self.scope_manager.get_declaration(node.id) - if declaration is None: - return None - else: - return {declaration} - - mac = MemberAccessCollector(node) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and len(mac.call_list) == 1: - declaration = self.scope_manager.get_declaration(node.func.id) - if declaration is None: - return None - else: - return {declaration} - elif isinstance(node, ast.Call) and len(mac.call_list) == 1 and not isinstance(node.func, ast.Subscript): - print("NEED FIX: () operator called on lambda, operator or await") - - mac = MemberAccessCollector(node) - last = mac.call_list.pop(0) - types = self.type_deduction.get_member_access_type(mac) - declarations = set() - - for d in types: - if isinstance(d, ModuleVariableDeclaration): - if isinstance(d.imported_module, GlobalScope): - declaration = self.scope_manager.get_declaration_from_scope(last.name, d.imported_module) - if declaration is not None: - declarations.add(declaration) - elif isinstance(d.imported_module, PreprocessedFile): - pass - elif d.imported_module is None: - pass - else: - assert False - elif isinstance(d, (VariableDeclaration, FunctionDeclaration)): - if isinstance(d, (TypeVariable, FunctionVariable)): - current_types = self.type_deduction.get_current_type({d.reference}) - else: - current_types = self.type_deduction.get_current_type(d.type) - for t in current_types: - if isinstance(t, ClassDeclaration): - if t is self.scope_manager.get_current_class_declaration(): - declarations.add(self.scope_manager.get_declaration(last.name)) - elif isinstance(last, - (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): - for m in t.attributes: - if m.name == last.name: - declarations.add(m) - for m in t.static_attributes: - if m.name == last.name: - declarations.add(m) - elif isinstance(last, MemberAccessCollector.MethodData): - for m in t.methods: - if m.name == last.name: - declarations.add(m) - for m in t.static_methods: - if m.name == last.name: - declarations.add(m) - else: - print("NEED FIX: BuiltIn, InitVariablePlaceholderType, VariablePlaceholderType") - elif isinstance(d, PlaceholderType): - pass - else: - assert False - - return declarations - - # TODO: change PlaceholderType-s, to actual type - # TODO: declaration is None -> what if not declared in global scope (eg. class member) - pass preprocessed 'scope'? - def post_process(self) -> None: - def post_process_variable(var: PreprocessedVariable) -> None: - var_declaration = self.scope_manager.get_global_scope().get_variable_by_name(var.name) - if var_declaration is None: - return - var_declaration.usages.extend(var.usages) - for type_usage in var.type_usages: - type_usage.type.update(var_declaration.type) - - def post_process_function(func: PreprocessedFunction) -> None: - func_declaration = self.scope_manager.get_global_scope().get_function_by_name(func.name) - if func_declaration is None: - return - func_declaration.usages.extend(func.usages) - for type_usage in func.type_usages: - type_usage.type.update(func_declaration.type) - - def post_process_class(cl: PreprocessedClass) -> None: - for cv in cl.attributes: - post_process_variable(cv) - for cf in cl.methods: - post_process_function(cf) - for cc in cl.classes: - post_process_class(cc) - - class_declaration = self.scope_manager.get_global_scope().get_class_by_name(cl.name) - if class_declaration is None: - return - for type_usage in cl.type_usages: - type_usage.type.update(class_declaration.type) - - for v in self.preprocessed_file.preprocessed_variables: - post_process_variable(v) - for f in self.preprocessed_file.preprocessed_functions: - post_process_function(f) - for c in self.preprocessed_file.class_collector.classes: - post_process_class(c) - - def create_var_usage(self, name: str, param: ast.expr) -> Usage: - r = create_range_from_ast_node(param) - return Usage(name, FilePosition(self.current_file, r)) - - def create_function_usage(self, func: ast.Call) -> Usage: - name = '' - h = ['func'] - while True: - if eval('hasattr(' + '.'.join(h) + ', "value")'): - if eval('isinstance(' + ".".join(h) + ', str)'): - break - else: - h.append('value') - elif eval('hasattr(' + '.'.join(h) + ', "func")'): - h.append('func') - else: - break - - n = [] - for i in range(len(h), 1, -1): - if eval('hasattr(' + '.'.join(h[:i]) + ', "id")'): - id_attr = "id" - elif eval('hasattr(' + '.'.join(h[:i]) + ', "attr")'): - id_attr = "attr" - else: - continue - n.append(eval("getattr(" + '.'.join(h[:i]) + ", '" + id_attr + "')")) - - # asdf = '.'.join(n) - - mac = MemberAccessCollector(func) - - if hasattr(func.func, 'id'): - name = func.func.id - elif hasattr(func.func, 'attr'): - name = func.func.attr - else: - print("NEED FIX: usage when () operator called on return value") - assert True # call on return value (eg. f()(), g[i]()) - - r = create_range_from_ast_node(func) - return Usage(name, FilePosition(self.current_file, r)) - - @staticmethod - def is_static_method(node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> bool: - for decorator in node.decorator_list: - if isinstance(decorator, ast.Name) and decorator.id == 'staticmethod': - return True - return False - - def is_static_variable(self, node: ast.AST) -> bool: - return isinstance(self.scope_manager.get_current_scope(), ClassScope) and isinstance(node, ast.Name) diff --git a/plugins/python/parser/src/my_ast/symbol_collector_interface.py b/plugins/python/parser/src/my_ast/symbol_collector_interface.py deleted file mode 100644 index 0cc617d5e..000000000 --- a/plugins/python/parser/src/my_ast/symbol_collector_interface.py +++ /dev/null @@ -1,27 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Dict - -from my_ast.base_data import ImportedDeclaration -from my_ast.import_finder import ImportFinder -from my_ast.member_access_collector import MemberAccessCollector -from my_ast.preprocessed_file import PreprocessedFile -from my_ast.scope_manager import ScopeManager - - -class ISymbolCollector(ABC): - @abstractmethod - def collect_symbols(self): - pass - - -class SymbolCollectorBase(ISymbolCollector, ABC): - def __init__(self, preprocessed_file: PreprocessedFile, import_finder: ImportFinder): - self.preprocessed_file: PreprocessedFile = preprocessed_file - self.import_finder: ImportFinder = import_finder - self.member_access_collector_type = MemberAccessCollector - self.imported_declaration_scope_map: Dict[ImportedDeclaration, ScopeManager] = {} - - -class IFunctionSymbolCollector(ISymbolCollector, ABC): - def __init__(self): - self.function = None diff --git a/plugins/python/parser/src/my_ast/symbol_finder.py b/plugins/python/parser/src/my_ast/symbol_finder.py deleted file mode 100644 index 23189bd15..000000000 --- a/plugins/python/parser/src/my_ast/symbol_finder.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Optional -from abc import ABC, abstractmethod - -from my_ast.base_data import Declaration - - -class SymbolFinder(ABC): - @abstractmethod - def get_declaration(self, name: str) -> Optional[Declaration]: - pass diff --git a/plugins/python/parser/src/my_ast/test.log b/plugins/python/parser/src/my_ast/test.log deleted file mode 100644 index eae8aac0b..000000000 --- a/plugins/python/parser/src/my_ast/test.log +++ /dev/null @@ -1,27 +0,0 @@ - -FILE: __init__ -========================================= -Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] - -FILE: asdf -========================================= - -FILE: qwer -========================================= -Func: g (line: 1, column: 0, length: 1) {global - } [] - -FILE: asdf -========================================= -Func: g (line: 0, column: 0, length: 0) {global - } [] -Func: f (line: 4, column: 0, length: 1) {global - } [] -Var: a (line: 4, column: 0, length: 1) {function - f} [Placeholder] -Var: b (line: 4, column: 0, length: 1) {function - f} [Placeholder] -Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] -Var Usage: a (line: 5, column: 10, length: 1) [Placeholder] -Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] -Var Usage: b (line: 6, column: 10, length: 1) [Placeholder] -Func Usage: g (line: 7, column: 4, length: 1) [] - -FILE: __init__ -========================================= -Var: __all__ (line: 1, column: 0, length: 7) {global - } [list] diff --git a/plugins/python/parser/src/my_ast/type_data.py b/plugins/python/parser/src/my_ast/type_data.py deleted file mode 100644 index 7e2956d26..000000000 --- a/plugins/python/parser/src/my_ast/type_data.py +++ /dev/null @@ -1,52 +0,0 @@ -from abc import ABC, abstractmethod - - -class DeclarationType(ABC): - def __init__(self, name: str, qualified_name: str): - self.name = name - self.qualified_name = qualified_name - - @abstractmethod - def get_type_repr(self) -> str: - pass - - @abstractmethod - def __hash__(self): - pass - - @abstractmethod - def __eq__(self, other): - pass - - @abstractmethod - def __ne__(self, other): - pass - - -class PlaceholderType(DeclarationType): - def __init__(self, name: str): - DeclarationType.__init__(self, name, "") - - def get_type_repr(self) -> str: - return 'Placeholder' - - def __hash__(self): - return hash(self.name) - - def __eq__(self, other): - return isinstance(other, type(self)) and self.name == other.name - - def __ne__(self, other): - return not self.__eq__(other) - - -class VariablePlaceholderType(PlaceholderType): - pass - - -class InitVariablePlaceholderType(VariablePlaceholderType): - pass - - -class FunctionPlaceholderType(PlaceholderType): - pass diff --git a/plugins/python/parser/src/my_ast/type_deduction.py b/plugins/python/parser/src/my_ast/type_deduction.py deleted file mode 100644 index a1d9569b0..000000000 --- a/plugins/python/parser/src/my_ast/type_deduction.py +++ /dev/null @@ -1,507 +0,0 @@ -import ast -import typing -from functools import singledispatchmethod -from typing import Optional, Any, Union - -from my_ast import built_in_types -from my_ast.base_data import Declaration, ImportedDeclaration, Usage -from my_ast.built_in_functions import get_built_in_function -from my_ast.built_in_operators import get_built_in_operator -from my_ast.built_in_types import Boolean, Dictionary, Set, Tuple, String, Integer, Float, Bytes, \ - EllipsisType, NoneType, Complex, RangeType, BuiltIn, Type, GenericBuiltInType -from my_ast.class_data import ClassDeclaration -from my_ast.common.file_position import FilePosition -from my_ast.common.hashable_list import OrderedHashableList -from my_ast.function_data import FunctionDeclaration -from my_ast.function_symbol_collector_factory import FunctionSymbolCollectorFactory -from my_ast.member_access_collector import MemberAccessCollector, MemberAccessCollectorIterator -from my_ast.preprocessed_file import PreprocessedFile -from my_ast.scope_manager import ScopeManager -from my_ast.symbol_collector_interface import SymbolCollectorBase -from my_ast.type_data import DeclarationType, PlaceholderType, VariablePlaceholderType, FunctionPlaceholderType,\ - InitVariablePlaceholderType -from my_ast.variable_data import VariableDeclaration, FunctionVariable, TypeVariable - - -class TypeDeduction: - def __init__(self, symbol_collector: SymbolCollectorBase, scope_manager: ScopeManager, - function_symbol_collector_factory: FunctionSymbolCollectorFactory): - self.scope_manager: ScopeManager = scope_manager - self.preprocessed_file: Optional[PreprocessedFile] = None - self.symbol_collector: SymbolCollectorBase = symbol_collector - self.function_symbol_collector_factory: FunctionSymbolCollectorFactory = function_symbol_collector_factory - - def deduct_type(self, node: ast.AST, preprocessed_file: PreprocessedFile) -> typing.Set: - self.preprocessed_file = preprocessed_file - types = self.get_type(node) - if not isinstance(types, typing.Set): - return {types} - return types - - def get_current_type(self, types: typing.Set[DeclarationType])\ - -> typing.Set[Union[ClassDeclaration, BuiltIn, FunctionVariable, TypeVariable, PlaceholderType]]: - def get_current_type_impl(declaration_type: DeclarationType)\ - -> typing.Set[Union[ClassDeclaration, BuiltIn, FunctionVariable, TypeVariable, PlaceholderType]]: - if isinstance(declaration_type, PlaceholderType): - if isinstance(declaration_type, InitVariablePlaceholderType): - return {declaration_type} - declaration = self.scope_manager.get_declaration(declaration_type.name) - if isinstance(declaration, PlaceholderType) and declaration_type.name == declaration.name: - return {declaration} - return self.get_current_type({declaration}) - elif isinstance(declaration_type, FunctionDeclaration): - return self.get_current_type(declaration_type.type) - elif isinstance(declaration_type, (ClassDeclaration, BuiltIn, FunctionVariable, TypeVariable)): - return {declaration_type} - elif isinstance(declaration_type, VariableDeclaration): - return declaration_type.type # VariableDeclaration.type should only be ClassDeclaration and BuiltIn - else: - assert False - - current_types = set() - for t in types: - if t is None: - continue - current_types.update(get_current_type_impl(t)) - return current_types - - @singledispatchmethod - def get_type(self, node: ast.AST) -> Any: - assert False, "Unknown type: " + str(type(node)) - - @get_type.register - def _(self, node: ast.BoolOp): - return Boolean() - - @get_type.register - def _(self, node: ast.NamedExpr): # (x := y) eg. if (y := f(x)) is not None: ... - pass - - @get_type.register - def _(self, node: ast.BinOp): - left_operand_type = self.get_current_type(self.deduct_type(node.left, self.preprocessed_file)) - bio = get_built_in_operator(node.op) - assert bio is not None - types = set() - for t in left_operand_type: - if isinstance(t, (BuiltIn, PlaceholderType)): - types.add(t) # TODO: handle numbers - elif isinstance(t, ClassDeclaration): - override = [x for x in t.methods if x.name in bio.get_override()] - if len(override) > 0: - types.add(override[0]) - else: - # TODO: check this (eg. base class is BuiltIn) - types.add(t) - else: - print("???") - return types # type(left_operand_type.op) - - @get_type.register - def _(self, node: ast.UnaryOp): - operand = self.get_current_type(self.deduct_type(node.operand, self.preprocessed_file)) - bio = get_built_in_operator(node.op) - assert bio is not None - types = set() - for t in operand: - if isinstance(t, BuiltIn): - types.add(t) - elif isinstance(t, ClassDeclaration): - override = [x for x in t.methods if x.name in bio.get_override()] - if len(override) > 0: - types.add(override[0]) - else: - assert False # TODO: is this a valid case? (operator on custom class without override it) - else: - print("???") - return types - - @get_type.register - def _(self, node: ast.Lambda): - # add node.args to tmp_scope - return self.get_type(node.body) - - @get_type.register - def _(self, node: ast.IfExp): - return {self.get_type(node.body), self.get_type(node.orelse)} - - @get_type.register - def _(self, node: ast.Dict): - return Dictionary(self.get_element_types(node)) - - @get_type.register - def _(self, node: ast.Set): - return Set(self.get_element_types(node)) - - @get_type.register - def _(self, node: ast.ListComp): - return built_in_types.List() # generic types? - - @get_type.register - def _(self, node: ast.SetComp): - return Set() # generic types? - - @get_type.register - def _(self, node: ast.DictComp): - return Dictionary() # generic types? - - @get_type.register - def _(self, node: ast.GeneratorExp): - pass # tmp_var := type(iter) -> type(target) - - @get_type.register - def _(self, node: ast.Await): - return self.get_type(node.expr) - - @get_type.register - def _(self, node: ast.Compare): - # TODO: handle PlaceHolderType - left_values = self.deduct_type(node.left, self.preprocessed_file) - types = set() - assert len(node.comparators) == len(node.ops) - for i in range(0, len(node.comparators)): - bio = get_built_in_operator(node.ops[i]) - assert bio is not None - left_operand_type = self.get_current_type(left_values) - for t in left_operand_type: - if isinstance(t, BuiltIn): - types.add(Boolean()) - elif isinstance(t, ClassDeclaration): - override = [x for x in t.methods if x.name in bio.get_override()] - if len(override) > 0: - types.add(override[0]) - else: - # TODO: check this (eg. base class is BuiltIn) - types.add(t) - else: - print("???") - left_values = types - return left_values - - @get_type.register - def _(self, node: ast.Call): - if hasattr(node.func, 'id'): - # TODO: check if builtin class ctor-s are not hidden by other (custom) functions/classes - # TODO: no args check (builtin is not hidden + correct code -> should not be a problem) - built_in_function = get_built_in_function(node.func.id) - if built_in_function is not None: - return built_in_function - elif node.func.id == 'TypeVar': - return Type() - return self.get_member_access_type(MemberAccessCollector(node)) - - @get_type.register - def _(self, node: ast.FormattedValue): # {x} in a joined string - return self.get_type(node.value) # cannot be on right side without f-string? - - @get_type.register - def _(self, node: ast.JoinedStr): # f"... {x} ... {y} ..." - return String() - - @get_type.register - def _(self, node: ast.Constant): - if isinstance(node.value, int): - if isinstance(node.value, bool): - return Boolean() - else: - return Integer() - elif isinstance(node.value, float): - return Float() - elif isinstance(node.value, bytes): - return Bytes() - elif isinstance(node.value, complex): - return Complex() - elif isinstance(node.value, str): - return String() - elif node.value is None: - return NoneType() - elif node.value is Ellipsis: - return EllipsisType() - return set() - - @get_type.register - def _(self, node: ast.Attribute): - return self.get_member_access_type(MemberAccessCollector(node)) - - @get_type.register - def _(self, node: ast.Subscript): - if isinstance(node.slice, ast.Index): - if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) - return Integer() - else: - if isinstance(node.value, ast.Name): - t = self.deduct_type(node.value, self.preprocessed_file) - elif isinstance(node.value, ast.Attribute): - t = self.deduct_type(node.value, self.preprocessed_file) - elif isinstance(node.value, ast.Subscript): - t = self.deduct_type(node.value, self.preprocessed_file) - else: - assert False - return set() # type(node.value->GenericType) - elif isinstance(node.slice, (ast.Slice, ast.ExtSlice)): # ExtSlice -> array type (module arr) - if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) - return RangeType() - else: - return set() # type(node.value)[type(node.value->GenericType)] - return set() # {type(value)...} - - @get_type.register - def _(self, node: ast.Starred): # unpack: * (eg. list, tuple) and ** (eg. dictionary) - return set() # could be same as iterator - - @get_type.register - def _(self, node: ast.Name): - return self.scope_manager.get_declaration(node.id) - - @get_type.register - def _(self, node: ast.List): - return built_in_types.List(self.get_element_types(node)) - - @get_type.register - def _(self, node: ast.Tuple): - return Tuple(self.get_element_types(node)) - - @get_type.register(ast.Return) - @get_type.register(ast.Yield) - @get_type.register(ast.YieldFrom) - def _(self, node): - return self.get_type_of_function(node) - - @get_type.register - def _(self, node: ast.Expr): - return self.get_type(node.value) - - def get_element_types(self, node: ast.AST): - element_types = set() - if isinstance(node, ast.Dict): - elements = getattr(node, 'values') - else: - assert hasattr(node, 'elts') - elements = getattr(node, 'elts') - for elem in elements: - elem_type = self.get_type(elem) - if elem_type is not None: - types = self.get_type(elem) - if isinstance(types, typing.Set): - element_types.update(types) - else: - element_types.add(types) - return element_types - - def get_member_access_type(self, mac: MemberAccessCollector): - if len(mac.call_list) == 0: - return set() - iterator = MemberAccessCollectorIterator(mac) - declaration = self.scope_manager.get_declaration_from_member_access(iterator) - if declaration is None or isinstance(declaration, PlaceholderType): - if iterator.is_iteration_started(): - a = iterator.get_current() - else: - a = iterator.get_first() - - # TODO: is empty FilePosition correct? - if isinstance(a, MemberAccessCollector.MethodData): - b = [x for x in self.preprocessed_file.preprocessed_functions if x.name == a.name] - c = [x for x in self.preprocessed_file.class_collector.classes if x.name == a.name] - for i in b: - i.usages.append(Usage(a.name, FilePosition.get_empty_file_position())) - for i in c: - i.usages.append(Usage(a.name, FilePosition.get_empty_file_position())) - elif isinstance(a, (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): - b = [x for x in self.preprocessed_file.preprocessed_variables if x.name == a.name] - for i in b: - i.usages.append(Usage(a.name, FilePosition.get_empty_file_position())) - declarations = {declaration} - else: - declarations = {declaration} - if iterator.is_iteration_over() or not iterator.is_iteration_started(): - if not any(isinstance(x, PlaceholderType) for x in declarations) and \ - any(any(isinstance(x, PlaceholderType) for x in y.type) for y in declarations): - declarations = self.get_current_type_of_placeholder_function(declarations, mac.call_list[-1]) - return declarations - prev_member = iterator.get_current() - next(iterator) - for member in mac.call_list[iterator.index::-1]: - if len(declarations) == 0 or all(x is None for x in declarations): - return set() - elif all(isinstance(x, PlaceholderType) for x in declarations): - return set() - declarations = self.get_member_declarations(declarations, prev_member, member) - prev_member = member - return declarations - - def get_member_declarations(self, declarations: typing.Set[DeclarationType], - current_member: MemberAccessCollector.MemberData, - next_member: MemberAccessCollector.MemberData) \ - -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: - member_declarations = set() - for declaration in declarations: - for declaration_type in self.get_current_type(declaration.type): - if isinstance(declaration_type, PlaceholderType) and isinstance(declaration, FunctionDeclaration): - d = self.get_current_type_of_placeholder_function({declaration}, current_member) - for dt in d: - for declaration_current_type in self.get_possible_types(dt): - if isinstance(declaration_current_type, BuiltIn): - continue - elif isinstance(declaration_current_type, ClassDeclaration): - member_declarations.update( - self.get_member(declaration_current_type, current_member, next_member)) - elif isinstance(declaration_current_type, VariablePlaceholderType): - continue # TODO: handle this (eg. function parameters) - else: - assert False - # member_declarations.update(d) - continue # TODO: try to get current value of function with the arguments from 'mac' - for declaration_current_type in self.get_possible_types(declaration_type): - if isinstance(declaration_current_type, BuiltIn): - self.get_member_of_builtin(declaration_current_type, current_member, next_member) - elif isinstance(declaration_current_type, ClassDeclaration): - member_declarations.update(self.get_member( - declaration_current_type, current_member, next_member)) - elif isinstance(declaration_current_type, VariablePlaceholderType): - continue # TODO: handle this (eg. function parameters) - elif isinstance(declaration_current_type, InitVariablePlaceholderType): - continue - else: - assert False - return member_declarations - - def get_member(self, declaration: Declaration, current_member: MemberAccessCollector.MemberData, - next_member: MemberAccessCollector.MemberData) \ - -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: - types = self.get_possible_types(declaration) - declarations = set() - for t in types: - if isinstance(t, ClassDeclaration): - if isinstance(next_member, (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): - for attr in reversed(t.attributes): - if attr.name == next_member.name: - declarations.add(attr) - for attr in reversed(t.static_attributes): - if attr.name == next_member.name: - declarations.add(attr) - elif isinstance(next_member, MemberAccessCollector.MethodData): - for method in reversed(t.methods): - if method.name == next_member.name: - declarations.add(method) - for method in reversed(t.static_methods): - if method.name == next_member.name: - declarations.add(method) - elif isinstance(next_member, MemberAccessCollector.SubscriptData): - # TODO: subscript elements type -> init/call method - pass - elif isinstance(next_member, MemberAccessCollector.LambdaData): - # TODO: member.node type -> init/call method - pass - elif isinstance(next_member, MemberAccessCollector.OperatorData): - # TODO: member.node type -> init/call method - pass - elif isinstance(t, BuiltIn): - continue - else: - assert False - return declarations - - def get_member_of_builtin(self, builtin: BuiltIn, current_member: MemberAccessCollector.MemberData, - next_member: MemberAccessCollector.MemberData) \ - -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: - if isinstance(builtin, GenericBuiltInType) and isinstance(current_member, MemberAccessCollector.SubscriptData): - return builtin.types - return set() - - # evaluate declaration to get type - def get_possible_types(self, declaration: DeclarationType) -> typing.Set[Union[ClassDeclaration, BuiltIn]]: - types = set() - # TODO: handle BuiltIn-s (Module!) - if isinstance(declaration, BuiltIn): - return {declaration} - elif isinstance(declaration, Declaration): - for t in declaration.type: - if isinstance(t, (ClassDeclaration, BuiltIn, PlaceholderType)): - types.add(t) - elif isinstance(t, (VariableDeclaration, FunctionDeclaration)): - types.update(self.get_possible_types(t)) - else: - assert False, "Unknown type: " + str(type(t)) - elif isinstance(declaration, PlaceholderType): - # TODO: get current value of function or variable - return self.get_current_type({self.scope_manager.get_declaration(declaration.name)}) - else: - assert False, "Unknown declaration type: " + str(type(declaration)) - return types - - def get_declaration(self, declaration: MemberAccessCollector.MemberData) \ - -> Optional[Union[VariableDeclaration, FunctionDeclaration]]: - # TODO: handle global, nonlocal and import declarations - return self.scope_manager.get_declaration(declaration.name) - - def get_current_type_of_placeholder_function(self, declarations: typing.Set, - member: MemberAccessCollector.MemberData) \ - -> typing.Set[DeclarationType]: - types = set() - for declaration in declarations: - arguments = OrderedHashableList() - if isinstance(member, MemberAccessCollector.MethodData): - for arg in member.arguments: - arguments.append(self.deduct_type(arg, self.preprocessed_file)) - declaration_type = self.scope_manager.placeholder_function_declaration_cache. \ - get_functions_return_type(declarations, arguments) - if declaration_type is not None and len(declaration_type) > 0: - types.update(declaration_type) - else: - if isinstance(declaration, ImportedDeclaration): - scope_manager = self.symbol_collector.imported_declaration_scope_map[declaration] - func_def = scope_manager.placeholder_function_declaration_cache. \ - get_function_def(declaration.imported_declaration) - else: - func_def = self.scope_manager.placeholder_function_declaration_cache.get_function_def(declaration) - # assert func_def is not None # VariableDeclaration? - if func_def is None: - continue - # magic - if len(arguments) == 0: - sc = self.function_symbol_collector_factory.get_function_symbol_collector( - self.symbol_collector, func_def, []) - sc.collect_symbols() - types.update(sc.function.type) - else: - for args in self.get_argument_combinations(arguments): - sc = self.function_symbol_collector_factory.get_function_symbol_collector( - self.symbol_collector, func_def, args) - sc.collect_symbols() - types.update(sc.function.type) - return types - - def get_argument_combinations(self, arguments: OrderedHashableList[typing.Set[DeclarationType]]) \ - -> typing.Iterator[OrderedHashableList[DeclarationType]]: - if len(arguments) == 0: - return OrderedHashableList() - for argument in arguments[0]: - head_argument = OrderedHashableList([argument]) - if len(arguments) == 1: - yield head_argument - else: - for tail_arguments in self.get_argument_combinations(arguments[1::]): - yield head_argument + tail_arguments - - def get_type_of_function(self, node: Union[ast.Return, ast.Yield, ast.YieldFrom]): - if node.value is None: - return set() - - types = self.get_type(node.value) - if types is None: - return set() - fixed_types = set() - if isinstance(types, typing.Set): - fixed_types.update(map(lambda x: self.fix_placeholder(x), types)) - else: - fixed_types.add(self.fix_placeholder(types)) - return fixed_types - - def fix_placeholder(self, declaration_type: DeclarationType): - if any(isinstance(x, PlaceholderType) for x in self.get_current_type({declaration_type})): - if isinstance(declaration_type, VariableDeclaration): - return VariablePlaceholderType(declaration_type.name) - elif isinstance(declaration_type, FunctionDeclaration): - return FunctionPlaceholderType(declaration_type.name) - return declaration_type diff --git a/plugins/python/parser/src/my_ast/variable_data.py b/plugins/python/parser/src/my_ast/variable_data.py deleted file mode 100644 index db4a40fd6..000000000 --- a/plugins/python/parser/src/my_ast/variable_data.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import PurePath -from typing import Optional, Set, Union - -from my_ast.built_in_types import Module, BuiltIn -from my_ast.common.file_position import FilePosition -from my_ast.base_data import Declaration, TypeDeclaration, ImportedDeclaration -from my_ast.function_data import FunctionDeclaration -from my_ast.persistence.base_dto import UsageDTO -from my_ast.persistence.variable_dto import VariableDeclarationDTO - - -class VariableDeclaration(Declaration): - # var_type: Optional[Set[Union[cd.ClassDeclaration, BuiltIn]]] - circular import - def __init__(self, name: str, qualified_name: str, pos: FilePosition, declaration_type: Optional[Set] = None): - super().__init__(name, qualified_name, pos, declaration_type) - # self.type: VariableDeclaration and FunctionDeclaration type can change - # (eg. new assignment with different type or redefinition) - - def create_dto(self) -> VariableDeclarationDTO: - usages = [] - for usage in self.usages: - usages.append(UsageDTO(usage.file_position)) - types = set() - for t in self.type: - types.add(t.qualified_name) - return VariableDeclarationDTO(self.name, self.qualified_name, self.file_position, types, usages) - - -class StaticVariableDeclaration(VariableDeclaration): - pass - - -class ReferenceVariableDeclaration(VariableDeclaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: Declaration): - super().__init__(name, qualified_name, pos, reference.type) - self.reference: Declaration = reference - - -class NonlocalVariableDeclaration(ReferenceVariableDeclaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): - super().__init__(name, qualified_name, pos, reference) - - -class GlobalVariableDeclaration(ReferenceVariableDeclaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): - super().__init__(name, qualified_name, pos, reference) - - -# TODO: short/long name? (in case of import) -class ModuleVariableDeclaration(VariableDeclaration): - # module: Union[GlobalScope, PreprocessedFile] - circular import - def __init__(self, name: str, location: PurePath, pos: FilePosition, module): - super().__init__(name, "", pos, {Module()}) # TODO: need qualified name? - self.imported_module = module - self.imported_module_location: PurePath = location - - -class ImportedVariableDeclaration(VariableDeclaration, ImportedDeclaration[VariableDeclaration]): - def __init__(self, name: str, pos: FilePosition, var_declaration: VariableDeclaration): - VariableDeclaration.__init__(self, name, "", pos, var_declaration.type) - ImportedDeclaration.__init__(self, var_declaration) - - -class TypeVariable(ReferenceVariableDeclaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, type_ref: TypeDeclaration): - super().__init__(name, qualified_name, pos, type_ref) - - def get_type_repr(self) -> str: - return '[TypeVariable(' + self.reference.name + ')]' - - -class FunctionVariable(ReferenceVariableDeclaration): - def __init__(self, name: str, qualified_name: str, pos: FilePosition, func_ref: FunctionDeclaration): - super().__init__(name, qualified_name, pos, func_ref) - - def get_type_repr(self) -> str: - return '[FunctionVariable(' + self.reference.name + ')]' diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 48e84582c..1955d4b7b 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -21,13 +21,21 @@ #include #include +#include #include +#include #include +#include #include +#include #include +#include #include +#include #include +#include #include +#include namespace cc { namespace parser{ @@ -53,6 +61,8 @@ class Persistence void Persistence::persistFile(boost::python::object pyFile) { + try{ + std::cout << "0" << std::endl; model::FilePtr file = nullptr; model::BuildSource buildSource; @@ -101,6 +111,9 @@ void Persistence::persistFile(boost::python::object pyFile) std::cout << "Exception: " << ex.what() << " - " << typeid(ex).name() << std::endl; } } +}catch(std::exception e){ + std::cout << e.what() << std::endl; +} // std::cout << std::endl << "START >>>" << std::endl; // // try{ @@ -216,6 +229,8 @@ void Persistence::persistFile(boost::python::object pyFile) void Persistence::persistVariable(boost::python::object pyVariable) { + try{ + std::cout << "1" << std::endl; boost::python::object name = pyVariable.attr("name"); boost::python::object qualifiedName = pyVariable.attr("qualified_name"); @@ -278,10 +293,15 @@ void Persistence::persistVariable(boost::python::object pyVariable) ctx.db->persist(usageAstNode); }); } +}catch(std::exception e){ + std::cout << e.what() << std::endl; +} } void Persistence::persistFunction(boost::python::object pyFunction) { + try{ + std::cout << "2" << std::endl; boost::python::object name = pyFunction.attr("name"); boost::python::object qualifiedName = pyFunction.attr("qualified_name"); @@ -355,10 +375,15 @@ void Persistence::persistFunction(boost::python::object pyFunction) ctx.db->persist(usageAstNode); }); } +}catch(std::exception e){ + std::cout << e.what() << std::endl; +} } void Persistence::persistClass(boost::python::object pyClass) { + try{ + std::cout << "3" << std::endl; boost::python::object name = pyClass.attr("name"); boost::python::object qualifiedName = pyClass.attr("qualified_name"); @@ -510,53 +535,61 @@ void Persistence::persistClass(boost::python::object pyClass) ctx.db->persist(classMember); }); } +}catch(std::exception e){ + std::cout << e.what() << std::endl; +} } void Persistence::persistImport(boost::python::object pyImport) { - model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); + try { + std::cout << "4" << std::endl; + model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); - boost::python::list importedModules = - boost::python::extract(pyImport.attr("imported_modules")); + boost::python::list importedModules = + boost::python::extract(pyImport.attr("imported_modules")); - boost::python::dict importedSymbols = - boost::python::extract(pyImport.attr("imported_symbols")); - - if(file == nullptr || importedModules.is_none() || importedSymbols.is_none()){ - return; - } + boost::python::dict importedSymbols = + boost::python::extract(pyImport.attr("imported_symbols")); - util::OdbTransaction transaction{ctx.db}; - - for(int i = 0; i(importedModules[i])); - if(moduleFile == nullptr){ - continue; + if (file == nullptr || importedModules.is_none() || importedSymbols.is_none()) { + return; } - model::PythonImportPtr moduleImport(new model::PythonImport); - moduleImport->importer = file; - moduleImport->imported = moduleFile; - transaction([&, this] { - ctx.db->persist(moduleImport); - }); - } + util::OdbTransaction transaction{ctx.db}; - boost::python::list importDict = importedSymbols.items(); - for(int i = 0; i(importDict[i]); - model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(import[0])); - if(moduleFile == nullptr){ - continue; + for (int i = 0; i < boost::python::len(importedModules); ++i) { + model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importedModules[i])); + if (moduleFile == nullptr) { + continue; + } + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->importer = file; + moduleImport->imported = moduleFile; + + transaction([&, this] { + ctx.db->persist(moduleImport); + }); } - model::PythonImportPtr moduleImport(new model::PythonImport); - moduleImport->importer = file; - moduleImport->imported = moduleFile; + + boost::python::list importDict = importedSymbols.items(); + for (int i = 0; i < boost::python::len(importDict); ++i) { + boost::python::tuple import = boost::python::extract(importDict[i]); + model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(import[0])); + if (moduleFile == nullptr) { + continue; + } + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->importer = file; + moduleImport->imported = moduleFile; // moduleImport->importedSymbol = getPythonEntityIdByQualifiedName(boost::python::extract(import[1])); - transaction([&, this] { - ctx.db->persist(moduleImport); - }); + transaction([&, this] { + ctx.db->persist(moduleImport); + }); + } + }catch(std::exception e){ + std::cout << e.what() << std::endl; } } diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt index aa75f0485..6d76cb904 100644 --- a/plugins/python/service/CMakeLists.txt +++ b/plugins/python/service/CMakeLists.txt @@ -18,7 +18,7 @@ target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) target_link_libraries(pythonservice util model - cppmodel + pythonmodel mongoose projectservice languagethrift diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 48a5753fe..2d16522bd 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include @@ -212,7 +214,7 @@ class PythonServiceHandler : virtual public LanguageServiceIf const odb::query& query_ = odb::query(true)); - std::vector queryDeclarations(const core::AstNodeId& astNodeId_) + std::vector queryDeclarations(const core::AstNodeId& astNodeId_); odb::query astCallsQuery(const model::PythonAstNode& astNode_); @@ -225,6 +227,10 @@ class PythonServiceHandler : virtual public LanguageServiceIf std::size_t queryCallsCount(const core::AstNodeId& astNodeId_); + std::vector queryTypes(const model::PythonEntity& entity); + + model::PythonEntity queryPythonEntity(const model::PythonEntityId& id); + std::shared_ptr _db; util::OdbTransaction _transaction; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 9e9488b5d..6e34a6f39 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -1,28 +1,31 @@ +#include + #include #include -#include -//#include +#include #include -//#include -#include -//#include +#include +#include #include -//#include +#include #include -//#include +#include #include -//#include +#include #include -//#include +#include #include -//#include +#include +#include namespace { typedef odb::query AstQuery; typedef odb::result AstResult; + typedef odb::query EntityQuery; + typedef odb::result EntityResult; typedef odb::query VarQuery; typedef odb::result VarResult; typedef odb::query FuncQuery; @@ -35,13 +38,17 @@ namespace typedef odb::result ModImpResult; typedef odb::query InhQuery; typedef odb::result InhResult; + typedef odb::query TypeQuery; + typedef odb::result TypeResult; + typedef odb::query FileQuery; + typedef odb::result FileResult; cc::service::language::AstNodeInfo createAstNodeInfo(const cc::model::PythonAstNode& astNode_) { cc::service::language::AstNodeInfo ret; ret.__set_id(std::to_string(astNode_.id)); - ret.__set_mangledNameHash(0); + ret.__set_entityHash(0); ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); ret.__set_astNodeValue(astNode_.astValue); @@ -67,7 +74,7 @@ PythonServiceHandler::PythonServiceHandler( std::shared_ptr db_, std::shared_ptr datadir_, const cc::webserver::ServerContext& context_) - : _db(db), + : _db(db_), _transaction(db_), _datadir(datadir_), _context(context_) @@ -153,7 +160,7 @@ void PythonServiceHandler::getDocumentation( { // The ast module does not include comments (~documentation). // TODO: try to use the tokenize module. - _return = std::string(); + return_ = std::string(); } void PythonServiceHandler::getProperties( @@ -173,7 +180,6 @@ void PythonServiceHandler::getProperties( return_["Name"] = variable.name; return_["Qualified name"] = variable.qualifiedName; - return_["Type"] = variable.qualifiedType; break; } @@ -184,7 +190,7 @@ void PythonServiceHandler::getProperties( model::PythonFunction function = *functions.begin(); return_["Name"] = function.qualifiedName.substr( - function.qualifiedName.find_last_of(':') + 1); + function.qualifiedName.find_last_of('.') + 1); return_["Qualified name"] = function.qualifiedName; return_["Signature"] = function.name; @@ -195,7 +201,7 @@ void PythonServiceHandler::getProperties( { ClassResult classes = _db->query( ClassQuery::astNodeId == node.id); - model::PythonCLass cl = *classes.begin(); + model::PythonClass cl = *classes.begin(); return_["Name"] = cl.name; return_["Qualified name"] = cl.qualifiedName; @@ -203,17 +209,18 @@ void PythonServiceHandler::getProperties( break; } - case model::PythonAstNode::SymbolType::Module: - { - ModImpResult modules = _db->query( - ModImpQuery::astNodeId == node.id); - model::PythonModule module = *modules.begin(); - - return_["Name"] = module.name; - return_["Qualified name"] = module.qualifiedName; - - break; - } +// case model::PythonAstNode::SymbolType::Module: +// { +// ModImpResult modules = _db->query( +// ModImpQuery::astNodeId == node.id); +// model::PythonImport module = *modules.begin(); +// +// return_["From"] = module.imported->filename; +// return_["To"] = module.importer->filename; +// //return_["Symbol"] = imported symbol +// +// break; +// } } }); } @@ -266,7 +273,7 @@ void PythonServiceHandler::getReferenceTypes( std::map& return_, const core::AstNodeId& astNodeId) { - model::PythonAstNode node = queryCppAstNode(astNodeId_); + model::PythonAstNode node = queryPythonAstNode(astNodeId); return_["Declaration"] = DECLARATION; return_["Usage"] = USAGE; @@ -334,7 +341,7 @@ void PythonServiceHandler::getReferences( for (const model::PythonAstNode& call : queryCalls(astNodeId_)) { core::AstNodeId astNodeId = std::to_string(call.id); - std::vector defs = queryDefinitions(astNodeId); + std::vector defs = queryDeclarations(astNodeId); nodes.insert(nodes.end(), defs.begin(), defs.end()); } @@ -346,7 +353,7 @@ void PythonServiceHandler::getReferences( case CALLER: for (const model::PythonAstNode& astNode : queryPythonAstNodes( astNodeId_, - AstQuery::astType == model::CppAstNode::AstType::Usage)) + AstQuery::astType == model::PythonAstNode::AstType::Usage)) { const model::Position& start = astNode.location.range.start; const model::Position& end = astNode.location.range.end; @@ -413,10 +420,9 @@ void PythonServiceHandler::getReferences( FuncQuery::astNodeId == node.id); model::PythonFunction function = *functions.begin(); - ClassResult result = _db->query( - ClassQuery::astNodeId == function.typeId); + std::vector types = queryTypes(function); - for (const model::PythonClass& cl : result) + for (const model::PythonClass& cl : types) { std::vector defs = queryDeclarations(std::to_string(cl.astNodeId)); @@ -435,10 +441,9 @@ void PythonServiceHandler::getReferences( const model::PythonVariable& variable = *varNodes.begin(); - ClassResult result = _db->query( - ClassQuery::astNodeId == variable.typeId); + std::vector types = queryTypes(variable); - for (const model::PythonClass& cl : result) + for (const model::PythonClass& cl : types) { std::vector defs = queryDeclarations(std::to_string(cl.astNodeId)); @@ -449,14 +454,16 @@ void PythonServiceHandler::getReferences( } case INHERIT_FROM: - node = queryCppAstNode(astNodeId_); + node = queryPythonAstNode(astNodeId_); for (const model::PythonInheritance& inh : _db->query( InhQuery::derived == node.id)) { + model::PythonEntity cl = _db->query_value( + EntityQuery::id == inh.base); AstResult result = _db->query( - AstQuery::astNodeId == inh.base && + AstQuery::id == cl.astNodeId && AstQuery::astType == model::PythonAstNode::AstType::Declaration); nodes.insert(nodes.end(), result.begin(), result.end()); } @@ -464,14 +471,16 @@ void PythonServiceHandler::getReferences( break; case INHERIT_BY: - node = queryCppAstNode(astNodeId_); + node = queryPythonAstNode(astNodeId_); for (const model::PythonInheritance& inh : _db->query( InhQuery::base == node.id )) { - AstResult result = _db->query( - AstQuery::astNodeId == inh.derived && + model::PythonEntity cl = _db->query_value( + EntityQuery::id == inh.base); + AstResult result = _db->query( + AstQuery::id == cl.astNodeId && AstQuery::astType == model::PythonAstNode::AstType::Declaration); nodes.insert(nodes.end(), result.begin(), result.end()); } @@ -484,12 +493,12 @@ void PythonServiceHandler::getReferences( for (const model::PythonClassMember& mem : _db->query( ClassMemQuery::astNodeId == node.id && - ClassMemQuery::kind == model::PythonClassMember::Kind::Field)) + ClassMemQuery::kind == model::PythonClassMember::Kind::Attribute)) { for (const model::PythonVariable& var : _db->query( VarQuery::id == mem.memberId)) { - model::PythonAstNode astNode = queryPythonAstNode(var.astNodeId); + model::PythonAstNode astNode = queryPythonAstNode(std::to_string(var.astNodeId)); if (astNode.location.range.end.line != model::Position::npos){ nodes.push_back(astNode); } @@ -507,10 +516,10 @@ void PythonServiceHandler::getReferences( ClassMemQuery::astNodeId == node.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Method)) { - for (const model::PythonFunction& var : _db->query( + for (const model::PythonFunction& func : _db->query( FuncQuery::id == mem.memberId)) { - nodes.push_back(queryPythonAstNode(var.astNodeId)); + nodes.push_back(queryPythonAstNode(std::to_string(func.astNodeId))); } } @@ -528,7 +537,7 @@ void PythonServiceHandler::getReferences( for (const model::PythonClass& cl : _db->query( ClassQuery::id == mem.memberId)) { - nodes.push_back(queryPythonAstNode(cl.astNodeId)); + nodes.push_back(queryPythonAstNode(std::to_string(cl.astNodeId))); } } @@ -559,7 +568,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( { case DECLARATION: return queryPythonAstNodeCount(astNodeId_, - AstQuery::astType == model::CppAstNode::AstType::Declaration); + AstQuery::astType == model::PythonAstNode::AstType::Declaration); case USAGE: return queryPythonAstNodeCount(astNodeId_); @@ -605,7 +614,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( FuncQuery::astNodeId == node.id).count; case LOCAL_VAR: - return _db->query_value( + return _db->query_value( FuncQuery::astNodeId == node.id).count; case RETURN_TYPE: @@ -617,10 +626,16 @@ std::int32_t PythonServiceHandler::getReferenceCount( const model::PythonFunction& function = *functions.begin(); - return _db->query_value( - ClassQuery::id == function.typeId).count; + std::vector types = queryTypes(function); - break; + std::int32_t result = 0; + + for(const model::PythonClass& cl : types){ + result += _db->query_value( + ClassQuery::id == function.id).count; + } + + return result; } case TYPE: @@ -632,8 +647,16 @@ std::int32_t PythonServiceHandler::getReferenceCount( const model::PythonVariable& variable = *varNodes.begin(); - return _db->query_value( - ClassQuery::id == variable.typeId).count; + std::vector types = queryTypes(variable); + + std::int32_t result = 0; + + for(const model::PythonClass& cl : types){ + result += _db->query_value( + ClassQuery::id == cl.id).count; + } + + return result; } case INHERIT_FROM: @@ -647,7 +670,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( case DATA_MEMBER: return _db->query_value( ClassMemQuery::astNodeId == node.id && - ClassMemQuery::kind == model::PythonClassMember::Kind::Field).count; + ClassMemQuery::kind == model::PythonClassMember::Kind::Attribute).count; case METHOD: return _db->query_value( @@ -707,24 +730,24 @@ void PythonServiceHandler::getFileReferences( { case CLASSES: nodes = queryPythonAstNodesInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Class); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Class); break; case VARIABLES: nodes = queryPythonAstNodesInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Variable && - AstQuery::astType == model::CppAstNode::AstType::Declaration); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Variable && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); break; case FUNCTIONS: nodes = queryPythonAstNodesInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Function && - (AstQuery::astType == model::CppAstNode::AstType::Declaration)); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + (AstQuery::astType == model::PythonAstNode::AstType::Declaration)); break; case IMPORTS: nodes = queryPythonAstNodesInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Import); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Module); break; } @@ -749,24 +772,24 @@ std::int32_t PythonServiceHandler::getFileReferenceCount( { case CLASSES: return queryPythonAstNodeCountInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Class); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Class); break; case VARIABLES: return queryPythonAstNodeCountInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Variable && - AstQuery::astType == model::CppAstNode::AstType::Declaration); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Variable && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); break; case FUNCTIONS: return queryPythonAstNodeCountInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Function && - AstQuery::astType == model::CppAstNode::AstType::Declaration)); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Function && + AstQuery::astType == model::PythonAstNode::AstType::Declaration); break; case IMPORTS: return queryPythonAstNodeCountInFile(fileId_, - AstQuery::symbolType == model::CppAstNode::SymbolType::Import); + AstQuery::symbolType == model::PythonAstNode::SymbolType::Module); break; default: @@ -882,7 +905,7 @@ std::vector PythonServiceHandler::queryPythonAstNodes( model::PythonAstNode node = queryPythonAstNode(astNodeId_); AstResult result = _db->query( - AstQuery::astNodeId == node.id && + AstQuery::id == node.id && AstQuery::location.range.end.line != model::Position::npos && query_); @@ -903,7 +926,7 @@ std::uint32_t PythonServiceHandler::queryPythonAstNodeCountInFile( const core::FileId& fileId_, const odb::query& query_) { - return _db->query_value( + return _db->query_value( AstQuery::location.file == std::stoull(fileId_) && query_).count; } @@ -929,7 +952,7 @@ odb::query PythonServiceHandler::astCallsQuery(const model // Pos > EndPos ((AstQuery::location.range.end.line == end.line && AstQuery::location.range.end.column < end.column) || - AstQuery::location.range.end.line < end.line)); + AstQuery::location.range.end.line < end.line))); } std::vector PythonServiceHandler::queryCalls(const core::AstNodeId& astNodeId_) @@ -953,14 +976,14 @@ std::size_t PythonServiceHandler::queryPythonAstNodeCount( model::PythonAstNode node = queryPythonAstNode(astNodeId_); model::PythonAstCount q = _db->query_value( - AstQuery::astNodeId == node.id && + AstQuery::id == node.id && AstQuery::location.range.end.line != model::Position::npos && query_); return q.count; } -std::size_t queryCallsCount(const core::AstNodeId& astNodeId_) +std::size_t PythonServiceHandler::queryCallsCount(const core::AstNodeId& astNodeId_) { std::vector nodes = queryDeclarations(astNodeId_); @@ -973,6 +996,24 @@ std::size_t queryCallsCount(const core::AstNodeId& astNodeId_) return _db->query_value(astCallsQuery(node)).count; } +std::vector PythonServiceHandler::queryTypes(const model::PythonEntity& entity) +{ + std::vector result; + + for(const model::PythonType& type : _db->query(TypeQuery::symbol == entity.id)){ + ClassResult cl = _db->query(ClassQuery::id == type.type); + result.push_back(*cl.begin()); + } + + return result; +} + +model::PythonEntity PythonServiceHandler::queryPythonEntity(const model::PythonEntityId& id) +{ + EntityResult entities = _db->query(EntityQuery::id == id); + return *entities.begin(); +} + } } } \ No newline at end of file From fd9cc1c36c33ab16729f3e68ba2b4f0c9aacdeb1 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 8 Nov 2020 20:39:31 +0100 Subject: [PATCH 03/39] Set ids to odb objects manually. --- .../model/include/model/pythonastnode.h | 31 + .../python/model/include/model/pythonentity.h | 16 +- plugins/python/parser/src/pythonparser.cpp | 714 ++++++++---------- 3 files changed, 371 insertions(+), 390 deletions(-) diff --git a/plugins/python/model/include/model/pythonastnode.h b/plugins/python/model/include/model/pythonastnode.h index d7e9fa1d6..3347c7c27 100644 --- a/plugins/python/model/include/model/pythonastnode.h +++ b/plugins/python/model/include/model/pythonastnode.h @@ -9,6 +9,8 @@ #include +#include + namespace cc { namespace model @@ -103,6 +105,35 @@ inline std::string PythonAstNode::toString() const .append("\nastType = ").append(astTypeToString(astType)); } +inline std::uint64_t createIdentifier(const PythonAstNode& astNode_) +{ + using SymbolTypeInt + = std::underlying_type::type; + using AstTypeInt + = std::underlying_type::type; + + std::string res; + + res + .append(astNode_.astValue).append(":") + .append(std::to_string(astNode_.qualifiedName)).append(":") + .append(std::to_string(static_cast(astNode_.symbolType))).append(":") + .append(std::to_string(static_cast(astNode_.astType))).append(":"); + + if (astNode_.location.file != nullptr){ + res + .append(std::to_string(astNode_.location.file->id)).append(":") + .append(std::to_string(astNode_.location.file->range.start.line)).append(":") + .append(std::to_string(astNode_.location.file->range.start.column)).append(":") + .append(std::to_string(astNode_.location.file->range.end.line)).append(":") + .append(std::to_string(astNode_.location.file->range.end.column)).append(":"); + } else { + res.append("null"); + } + + return util::fnvHash(res); +} + #pragma db view \ object(PythonAstNode) object(File = LocFile : PythonAstNode::location.file) \ query ((?) + "GROUP BY" + LocFile::id + "ORDER BY" + LocFile::id) diff --git a/plugins/python/model/include/model/pythonentity.h b/plugins/python/model/include/model/pythonentity.h index cbc8245c8..e6e71d34e 100644 --- a/plugins/python/model/include/model/pythonentity.h +++ b/plugins/python/model/include/model/pythonentity.h @@ -5,6 +5,8 @@ #include "pythonastnode.h" +#include + namespace cc { namespace model @@ -17,7 +19,7 @@ struct PythonEntity { virtual ~PythonEntity() {} - #pragma db id auto + #pragma db id PythonEntityId id; #pragma db unique @@ -29,6 +31,18 @@ struct PythonEntity typedef std::shared_ptr PythonEntityPtr; +inline PythonEntityId createIdentifier(const PythonEntity& entity_) +{ + std::string res; + + res + .append(std::to_string(entity_.astNodeId)).append(":") + .append(entity_.name).append(":") + .append(entity_.qualifiedName).append(":"); + + return util::fnvHash(res); +} + } } diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 1955d4b7b..efcc8d1bc 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -57,487 +57,389 @@ class Persistence private: boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); + model::PythonEntity* getPythonEntity(const std::string& qualifiedName, const model::FileLoc& fileLoc); }; void Persistence::persistFile(boost::python::object pyFile) { try{ - std::cout << "0" << std::endl; - model::FilePtr file = nullptr; - model::BuildSource buildSource; + std::cout << "0" << std::endl; + model::FilePtr file = nullptr; + model::BuildSource buildSource; - boost::python::object path = pyFile.attr("path"); - boost::python::object status = pyFile.attr("parse_status"); + boost::python::object path = pyFile.attr("path"); + boost::python::object status = pyFile.attr("parse_status"); - if(status.is_none()){ - std::cout << "status is None..." << std::endl; - } else if(path.is_none()){ - std::cout << "path is None..." << std::endl; - } else { - buildSource.file = ctx.srcMgr.getFile(boost::python::extract(path)); - std::cout << buildSource.file << std::endl; - switch(boost::python::extract(status)){ - case 0: - buildSource.file->parseStatus = model::File::PSNone; - break; - case 1: - buildSource.file->parseStatus = model::File::PSPartiallyParsed; - break; - case 2: - buildSource.file->parseStatus = model::File::PSFullyParsed; - break; - default: - std::cout << "Unknown status: " << boost::python::extract(status) << std::endl; - } + if(status.is_none()){ + std::cout << "status is None..." << std::endl; + } else if(path.is_none()){ + std::cout << "path is None..." << std::endl; + } else { + buildSource.file = ctx.srcMgr.getFile(boost::python::extract(path)); + std::cout << buildSource.file << std::endl; + switch(boost::python::extract(status)){ + case 0: + buildSource.file->parseStatus = model::File::PSNone; + break; + case 1: + buildSource.file->parseStatus = model::File::PSPartiallyParsed; + break; + case 2: + buildSource.file->parseStatus = model::File::PSFullyParsed; + break; + default: + std::cout << "Unknown status: " << boost::python::extract(status) << std::endl; + } - model::BuildActionPtr buildAction(new model::BuildAction); - buildAction->command = ""; - buildAction->type = model::BuildAction::Other; - try{ - util::OdbTransaction transaction{ctx.db}; + model::BuildActionPtr buildAction(new model::BuildAction); + buildAction->command = ""; + buildAction->type = model::BuildAction::Other; + try{ + util::OdbTransaction transaction{ctx.db}; - transaction([&, this] { - ctx.db->persist(buildAction); - }); + transaction([&, this] { + ctx.db->persist(buildAction); + }); - buildSource.action = buildAction; + buildSource.action = buildAction; - ctx.srcMgr.updateFile(*buildSource.file); + ctx.srcMgr.updateFile(*buildSource.file); - transaction([&, this] { - ctx.db->persist(buildSource); - }); - } catch(const std::exception& ex){ - std::cout << "Exception: " << ex.what() << " - " << typeid(ex).name() << std::endl; + transaction([&, this] { + ctx.db->persist(buildSource); + }); + } catch(const std::exception& ex){ + std::cout << "Exception: " << ex.what() << " - " << typeid(ex).name() << std::endl; + } } + } catch(std::exception e){ + std::cout << e.what() << std::endl; } -}catch(std::exception e){ - std::cout << e.what() << std::endl; -} -// std::cout << std::endl << "START >>>" << std::endl; -// -// try{ -// boost::python::object file_name = pyFile.attr("file_name"); -// -// if(file_name.is_none()){ -// std::cout << "file name is None..." << std::endl; -// } else { -// std::string s = boost::python::extract(file_name); -// std::cout << "File name: " << s << std::endl; -// file.filename = s; -// } -// -// boost::python::object type = pyFile.attr("type"); -// -// if(type.is_none()){ -// std::cout << "type name is None..." << std::endl; -// } else { -// std::string s = boost::python::extract(type); -// std::cout << "Type: " << s << std::endl; -// file.type = s; -// } -// -// boost::python::object path = pyFile.attr("path"); -// -// if(path.is_none()){ -// std::cout << "path is None..." << std::endl; -// } else { -// std::string s = boost::python::extract(path); -// std::cout << "Path: " << s << std::endl; -// -// boost::system::error_code ec; -// boost::filesystem::path p(s); -// std::time_t timestamp = boost::filesystem::last_write_time(p, ec); -// if(ec){ -// timestamp = 0; -// } -// -// file.path = s; -// file.id = util::fnvHash(s); -// file.timestamp = timestamp; -// } -// -// boost::python::object directory = pyFile.attr("parent"); -// -// if(directory.is_none()){ -// std::cout << "directory is None..." << std::endl; -// } else { -// std::string s = boost::python::extract(directory); -// std::cout << "Parent folder: " << s << std::endl; -// -// boost::filesystem::path parentPath = boost::filesystem::path(s); -// -// if (parentPath.native().empty()){ -// file.parent = nullptr; -// } else { -// file.parent = nullptr; // getFile(parentPath.native()); -// } -// } -// -// boost::python::object content_wrapper = pyFile.attr("content"); -// -// if(content_wrapper.is_none()){ -// std::cout << "content_wrapper is None..." << std::endl; -// } else { -// boost::python::object content = content_wrapper.attr("content"); -// -// if(content.is_none()){ -// std::cout << "content is None..." << std::endl; -// } else { -// std::string s = boost::python::extract(content); -// std::cout << "Content:\n" << s << std::endl; -// -// model::FileContentPtr file_content(new model::FileContent); -// -// file_content->hash = util::sha1Hash(s); -// file_content->content = s; -// -// file.content = file_content; -// } -// } -// -// boost::python::object status = pyFile.attr("parse_status"); -// -// if(status.is_none()){ -// std::cout << "status is None..." << std::endl; -// } else { -// int s = boost::python::extract(status); -// std::cout << "Status: " << s << std::endl; -// -// switch(s){ -// case 0: -// file.parseStatus = model::File::PSNone; -// break; -// case 1: -// file.parseStatus = model::File::PSPartiallyParsed; -// break; -// case 2: -// file.parseStatus = model::File::PSFullyParsed; -// break; -// default: -// std::cout << "Unknown status: " << s << std::endl; -// } -// } -// } catch(const boost::python::error_already_set&) { -// std::cout << "error_already_set" << std::endl; -// } catch(const std::exception& ex) { -// std::cout << ex.what() << std::endl; -// } - -// std::cout << "<<< END" << std::endl << std::endl; } void Persistence::persistVariable(boost::python::object pyVariable) { try{ - std::cout << "1" << std::endl; - boost::python::object name = pyVariable.attr("name"); - boost::python::object qualifiedName = pyVariable.attr("qualified_name"); + std::cout << "1" << std::endl; + boost::python::object name = pyVariable.attr("name"); + boost::python::object qualifiedName = pyVariable.attr("qualified_name"); - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyVariable.attr("file_position")); + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyVariable.attr("file_position")); - // set - boost::python::list types = boost::python::extract(pyVariable.attr("type")); + // set + boost::python::list types = boost::python::extract(pyVariable.attr("type")); - boost::python::list usages = boost::python::extract(pyVariable.attr("usages")); + boost::python::list usages = boost::python::extract(pyVariable.attr("usages")); - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || - types.is_none() || usages.is_none()){ - return; - } + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + types.is_none() || usages.is_none()){ + return; + } - util::OdbTransaction transaction{ctx.db}; + util::OdbTransaction transaction{ctx.db}; - model::PythonAstNodePtr varAstNode(new model::PythonAstNode); - varAstNode->location = fileLoc.get(); - varAstNode->qualifiedName = boost::python::extract(qualifiedName); - varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; - varAstNode->astType = model::PythonAstNode::AstType::Declaration; + model::PythonAstNodePtr varAstNode(new model::PythonAstNode); + varAstNode->location = fileLoc.get(); + varAstNode->qualifiedName = boost::python::extract(qualifiedName); + varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + varAstNode->astType = model::PythonAstNode::AstType::Declaration; - transaction([&, this] { - ctx.db->persist(varAstNode); - }); + varAstNode->id = model::createIdentifier(*varAstNode); - model::PythonVariablePtr variable(new model::PythonVariable); -// variable->astNodeId = ? need to insert varAstNode first? - variable->name = boost::python::extract(name); - variable->qualifiedName = boost::python::extract(qualifiedName); + transaction([&, this] { + ctx.db->persist(varAstNode); + }); - transaction([&, this] { - ctx.db->persist(variable); - }); + model::PythonVariablePtr variable(new model::PythonVariable); + variable->astNodeId = varAstNode->id; + variable->name = boost::python::extract(name); + variable->qualifiedName = boost::python::extract(qualifiedName); - for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); -// type->symbol = id of variable + variable->id = model::createIdentifier(*variable); transaction([&, this] { - ctx.db->persist(type); + ctx.db->persist(variable); }); - } - for(int i = 0; i usageFileLoc = - createFileLocFromPythonFilePosition(usages[i].attr("file_position")); - if(usageFileLoc == boost::none){ - continue; + for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); + type->symbol = variable->id; + + transaction([&, this] { + ctx.db->persist(type); + }); } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(usageAstNode); - }); + for(int i = 0; i usageFileLoc = + createFileLocFromPythonFilePosition(usages[i].attr("file_position")); + if(usageFileLoc == boost::none){ + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + transaction([&, this] { + ctx.db->persist(usageAstNode); + }); + } + } catch(std::exception e){ + std::cout << e.what() << std::endl; } -}catch(std::exception e){ - std::cout << e.what() << std::endl; -} } void Persistence::persistFunction(boost::python::object pyFunction) { try{ - std::cout << "2" << std::endl; - boost::python::object name = pyFunction.attr("name"); - boost::python::object qualifiedName = pyFunction.attr("qualified_name"); + std::cout << "2" << std::endl; + boost::python::object name = pyFunction.attr("name"); + boost::python::object qualifiedName = pyFunction.attr("qualified_name"); - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyFunction.attr("file_position")); + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyFunction.attr("file_position")); - // set - boost::python::list types = boost::python::extract(pyFunction.attr("type")); + // set + boost::python::list types = boost::python::extract(pyFunction.attr("type")); - boost::python::list usages = boost::python::extract(pyFunction.attr("usages")); + boost::python::list usages = boost::python::extract(pyFunction.attr("usages")); - boost::python::object pyDocumentation = pyFunction.attr("documentation"); + boost::python::object pyDocumentation = pyFunction.attr("documentation"); - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || - types.is_none() || usages.is_none() || pyDocumentation.is_none()){ - return; - } + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + types.is_none() || usages.is_none() || pyDocumentation.is_none()){ + return; + } - util::OdbTransaction transaction{ctx.db}; + util::OdbTransaction transaction{ctx.db}; - model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); - funcAstNode->location = fileLoc.get(); - funcAstNode->qualifiedName = boost::python::extract(qualifiedName); - funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; - funcAstNode->astType = model::PythonAstNode::AstType::Declaration; + model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); + funcAstNode->location = fileLoc.get(); + funcAstNode->qualifiedName = boost::python::extract(qualifiedName); + funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + funcAstNode->astType = model::PythonAstNode::AstType::Declaration; - transaction([&, this] { - ctx.db->persist(funcAstNode); - }); + funcAstNode->id = model::createIdentifier(*funcAstNode); - model::PythonFunctionPtr function(new model::PythonFunction); -// function->astNodeId = ? need to insert varAstNode first? - function->name = boost::python::extract(name); - function->qualifiedName = boost::python::extract(qualifiedName); + transaction([&, this] { + ctx.db->persist(funcAstNode); + }); - transaction([&, this] { - ctx.db->persist(function); - }); + model::PythonFunctionPtr function(new model::PythonFunction); + function->astNodeId = funcAstNode->id; + function->name = boost::python::extract(name); + function->qualifiedName = boost::python::extract(qualifiedName); - model::PythonDocumentationPtr documentation(new model::PythonDocumentation); - documentation->documentation = boost::python::extract(pyDocumentation); -// documentation->documented = id of function - documentation->documentationKind = model::PythonDocumentation::Function; + function->id = model::createIdentifier(*function); - transaction([&, this] { - ctx.db->persist(documentation); - }); + transaction([&, this] { + ctx.db->persist(function); + }); - for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); -// type->symbol = id of function + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = boost::python::extract(pyDocumentation); + documentation->documented = function->id; + documentation->documentationKind = model::PythonDocumentation::Function; transaction([&, this] { - ctx.db->persist(type); + ctx.db->persist(documentation); }); - } - for(int i = 0; i usageFileLoc = - createFileLocFromPythonFilePosition(usages[i].attr("file_position")); - if(usageFileLoc == boost::none){ - continue; + for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); + type->symbol = function->id; + + transaction([&, this] { + ctx.db->persist(type); + }); } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(usageAstNode); - }); + for(int i = 0; i usageFileLoc = + createFileLocFromPythonFilePosition(usages[i].attr("file_position")); + if(usageFileLoc == boost::none){ + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + transaction([&, this] { + ctx.db->persist(usageAstNode); + }); + } + } catch(std::exception e){ + std::cout << e.what() << std::endl; } -}catch(std::exception e){ - std::cout << e.what() << std::endl; -} } void Persistence::persistClass(boost::python::object pyClass) { try{ - std::cout << "3" << std::endl; - boost::python::object name = pyClass.attr("name"); - boost::python::object qualifiedName = pyClass.attr("qualified_name"); + std::cout << "3" << std::endl; + boost::python::object name = pyClass.attr("name"); + boost::python::object qualifiedName = pyClass.attr("qualified_name"); - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyClass.attr("file_position")); + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyClass.attr("file_position")); - boost::python::list usages = boost::python::extract(pyClass.attr("usages")); + boost::python::list usages = boost::python::extract(pyClass.attr("usages")); - boost::python::object pyDocumentation = pyClass.attr("documentation"); + boost::python::object pyDocumentation = pyClass.attr("documentation"); - // set - boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); + // set + boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); - boost::python::object members = pyClass.attr("members"); + boost::python::object members = pyClass.attr("members"); - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || - usages.is_none() || pyDocumentation.is_none() || baseClasses.is_none() || members.is_none()){ - return; - } - - util::OdbTransaction transaction{ctx.db}; - - model::PythonAstNodePtr classAstNode(new model::PythonAstNode); - classAstNode->location = fileLoc.get(); - classAstNode->qualifiedName = boost::python::extract(qualifiedName); - classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; - classAstNode->astType = model::PythonAstNode::AstType::Declaration; - - transaction([&, this] { - ctx.db->persist(classAstNode); - }); - - model::PythonClassPtr cl(new model::PythonClass); -// function->astNodeId = ? need to insert varAstNode first? - cl->name = boost::python::extract(name); - cl->qualifiedName = boost::python::extract(qualifiedName); - - transaction([&, this] { - ctx.db->persist(cl); - }); - - model::PythonDocumentationPtr documentation(new model::PythonDocumentation); - documentation->documentation = boost::python::extract(pyDocumentation); -// documentation->documented = id of function - documentation->documentationKind = model::PythonDocumentation::Class; - - transaction([&, this] { - ctx.db->persist(documentation); - }); - - for(int i = 0; i usageFileLoc = - createFileLocFromPythonFilePosition(usages[i].attr("file_position")); - if(usageFileLoc == boost::none){ - continue; + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + usages.is_none() || pyDocumentation.is_none() || baseClasses.is_none() || members.is_none()){ + return; } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(usageAstNode); - }); - } + util::OdbTransaction transaction{ctx.db}; - for(int i = 0; iderived = id of cl -// inheritance->base = getPythonEntityIdByQualifiedName(boost::python::extract(baseClasses[i])); + model::PythonAstNodePtr classAstNode(new model::PythonAstNode); + classAstNode->location = fileLoc.get(); + classAstNode->qualifiedName = boost::python::extract(qualifiedName); + classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + classAstNode->astType = model::PythonAstNode::AstType::Declaration; + + classAstNode->id = model::createIdentifier(*classAstNode); transaction([&, this] { - ctx.db->persist(inheritance); + ctx.db->persist(classAstNode); }); - } - boost::python::list methods = boost::python::extract(members.attr("methods")); - boost::python::list staticMethods = boost::python::extract(members.attr("static_methods")); - boost::python::list attributes = boost::python::extract(members.attr("attributes")); - boost::python::list staticAttributes = boost::python::extract(members.attr("static_attributes")); - boost::python::list classes = boost::python::extract(members.attr("classes")); - - if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || - staticAttributes.is_none() || classes.is_none()){ - return; - } + model::PythonClassPtr cl(new model::PythonClass); + cl->astNodeId = classAstNode->id; + cl->name = boost::python::extract(name); + cl->qualifiedName = boost::python::extract(qualifiedName); - for(int i = 0; iastNodeId = id of classAstNode -// classMember->memberId = search for function declaration -// classMember->classId = id of cl - classMember->kind = model::PythonClassMember::Method; - classMember->staticMember = false; + cl->id = model::creatIdentifier(*cl); transaction([&, this] { - ctx.db->persist(classMember); + ctx.db->persist(cl); }); - } - for(int i = 0; iastNodeId = id of classAstNode -// classMember->memberId = search for function declaration -// classMember->classId = id of cl - classMember->kind = model::PythonClassMember::Method; - classMember->staticMember = true; + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = boost::python::extract(pyDocumentation); + documentation->documented = cl->id; + documentation->documentationKind = model::PythonDocumentation::Class; transaction([&, this] { - ctx.db->persist(classMember); + ctx.db->persist(documentation); }); - } - for(int i = 0; iastNodeId = id of classAstNode -// classMember->memberId = search for variable declaration -// classMember->classId = id of cl - classMember->kind = model::PythonClassMember::Attribute; - classMember->staticMember = false; + for(int i = 0; i usageFileLoc = + createFileLocFromPythonFilePosition(usages[i].attr("file_position")); + if(usageFileLoc == boost::none){ + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(classMember); - }); - } + transaction([&, this] { + ctx.db->persist(usageAstNode); + }); + } - for(int i = 0; iastNodeId = id of classAstNode -// classMember->memberId = search for variable declaration -// classMember->classId = id of cl - classMember->kind = model::PythonClassMember::Attribute; - classMember->staticMember = true; + for(int i = 0; iderived = cl->id; + // inheritance->base = getPythonEntityIdByQualifiedName(boost::python::extract(baseClasses[i])); - transaction([&, this] { - ctx.db->persist(classMember); - }); - } + transaction([&, this] { + ctx.db->persist(inheritance); + }); + } - for(int i = 0; iastNodeId = id of classAstNode -// classMember->memberId = search for class declaration -// classMember->classId = id of cl - classMember->kind = model::PythonClassMember::Class; - classMember->staticMember = false; + boost::python::list methods = boost::python::extract(members.attr("methods")); + boost::python::list staticMethods = boost::python::extract(members.attr("static_methods")); + boost::python::list attributes = boost::python::extract(members.attr("attributes")); + boost::python::list staticAttributes = boost::python::extract(members.attr("static_attributes")); + boost::python::list classes = boost::python::extract(members.attr("classes")); - transaction([&, this] { - ctx.db->persist(classMember); - }); + if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || + staticAttributes.is_none() || classes.is_none()){ + return; + } + + for(int i = 0; iastNodeId = classAstNode->id; + // classMember->memberId = search for function declaration + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = false; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = classAstNode->id; + // classMember->memberId = search for function declaration + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = true; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = classAstNode->id; + // classMember->memberId = search for variable declaration + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = false; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = classAstNode->id; + // classMember->memberId = search for variable declaration + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = true; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + + for(int i = 0; iastNodeId = classAstNode->id; + // classMember->memberId = search for class declaration + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Class; + classMember->staticMember = false; + + transaction([&, this] { + ctx.db->persist(classMember); + }); + } + } catch(std::exception e){ + std::cout << e.what() << std::endl; } -}catch(std::exception e){ - std::cout << e.what() << std::endl; -} } void Persistence::persistImport(boost::python::object pyImport) @@ -582,13 +484,13 @@ void Persistence::persistImport(boost::python::object pyImport) model::PythonImportPtr moduleImport(new model::PythonImport); moduleImport->importer = file; moduleImport->imported = moduleFile; -// moduleImport->importedSymbol = getPythonEntityIdByQualifiedName(boost::python::extract(import[1])); +// moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1])); transaction([&, this] { ctx.db->persist(moduleImport); }); } - }catch(std::exception e){ + } catch(std::exception e){ std::cout << e.what() << std::endl; } } @@ -627,6 +529,40 @@ boost::optional Persistence::createFileLocFromPythonFilePosition return fileLoc; } +model::PythonEntity* Persistence::getPythonEntity(const std::string& qualifiedName, const model::FileLoc& fileLoc) +{ + using odb::query AstQuery; + using odb::result AstResult; + using odb::query EntityQuery; + using odb::result EntityResult; + + const model::Position& start = astNode_.location.range.start; + const model::Position& end = astNode_.location.range.end; + + AstResult astNode = ctx.db->query( + AstQuery::location.file == astNode_.location.file.object_id() && + // StartPos == Pos + (AstQuery::location.range.start.line == start.line && + AstQuery::location.range.start.column == start.column) && + // Pos == EndPos + (AstQuery::location.range.end.line == end.line && + AstQuery::location.range.end.column == end.column)); + + if (astNode.empty()){ + return nullptr; + } + + EntityResult entity = ctx.db->query( + EntityQuery::qualifiedName == qualifiedName && + EntityQuery::astNodeId == astNode.begin()->id); + + if (entity.empty()){ + return nullptr; + } + + return entity.begin(); +} + typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ From 29a485800b866eb2f50c0d294cc4676e0e4d7b81 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 8 Nov 2020 21:15:14 +0100 Subject: [PATCH 04/39] Python service getDocumentation implementation. --- .../service/include/service/pythonservice.h | 2 ++ plugins/python/service/src/pythonservice.cpp | 21 ++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 2d16522bd..585f12fc3 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -231,6 +231,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf model::PythonEntity queryPythonEntity(const model::PythonEntityId& id); + model::PythonEntity queryPythonEntityByAstNode(const model::AstNodeId& id); + std::shared_ptr _db; util::OdbTransaction _transaction; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 6e34a6f39..5dbdffc07 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -40,6 +40,8 @@ namespace typedef odb::result InhResult; typedef odb::query TypeQuery; typedef odb::result TypeResult; + typedef odb::query DocQuery; + typedef odb::result DocResult; typedef odb::query FileQuery; typedef odb::result FileResult; @@ -158,9 +160,16 @@ void PythonServiceHandler::getDocumentation( std::string& return_, const core::AstNodeId& astNodeId_) { - // The ast module does not include comments (~documentation). - // TODO: try to use the tokenize module. - return_ = std::string(); + PythonEntity entity = queryPythonEntityByAstNode(astNodeId_); + + DocResult doc = _db->query( + DocQuery::documented == entity.id); + + if (doc.empty()){ + return std::string(); + } + + return_ = doc.begin()->documentation; } void PythonServiceHandler::getProperties( @@ -1014,6 +1023,12 @@ model::PythonEntity PythonServiceHandler::queryPythonEntity(const model::PythonE return *entities.begin(); } +model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const model::AstNodeId& id) +{ + EntityResult entities = _db->query(EntityQuery::astNodeId == id); + return *entities.begin(); +} + } } } \ No newline at end of file From 1d4344c24fa0fa2251314ae63e2af64360105c71 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Tue, 10 Nov 2020 16:41:53 +0100 Subject: [PATCH 05/39] Introduce visibility in PythonEntity --- .../python/model/include/model/pythonentity.h | 1 + plugins/python/parser/src/pythonparser.cpp | 7 ++ .../service/include/service/pythonservice.h | 4 + plugins/python/service/src/pythonservice.cpp | 98 +++++++++++++------ 4 files changed, 79 insertions(+), 31 deletions(-) diff --git a/plugins/python/model/include/model/pythonentity.h b/plugins/python/model/include/model/pythonentity.h index e6e71d34e..c9c7bbd13 100644 --- a/plugins/python/model/include/model/pythonentity.h +++ b/plugins/python/model/include/model/pythonentity.h @@ -27,6 +27,7 @@ struct PythonEntity std::string name; std::string qualifiedName; + std::string visibility; }; typedef std::shared_ptr PythonEntityPtr; diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index efcc8d1bc..f718cce0b 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -123,6 +123,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) std::cout << "1" << std::endl; boost::python::object name = pyVariable.attr("name"); boost::python::object qualifiedName = pyVariable.attr("qualified_name"); + boost::python::object visibility = pyVariable.attr("visibility"); boost::optional fileLoc = createFileLocFromPythonFilePosition(pyVariable.attr("file_position")); @@ -154,6 +155,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) variable->astNodeId = varAstNode->id; variable->name = boost::python::extract(name); variable->qualifiedName = boost::python::extract(qualifiedName); + variable->visibility = boost::python::extract(visibility); variable->id = model::createIdentifier(*variable); @@ -198,6 +200,8 @@ void Persistence::persistFunction(boost::python::object pyFunction) std::cout << "2" << std::endl; boost::python::object name = pyFunction.attr("name"); boost::python::object qualifiedName = pyFunction.attr("qualified_name"); + boost::python::object visibility = pyFunction.attr("visibility"); + boost::optional fileLoc = createFileLocFromPythonFilePosition(pyFunction.attr("file_position")); @@ -231,6 +235,7 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->astNodeId = funcAstNode->id; function->name = boost::python::extract(name); function->qualifiedName = boost::python::extract(qualifiedName); + function->visibility = boost::python::extract(visibility); function->id = model::createIdentifier(*function); @@ -284,6 +289,7 @@ void Persistence::persistClass(boost::python::object pyClass) std::cout << "3" << std::endl; boost::python::object name = pyClass.attr("name"); boost::python::object qualifiedName = pyClass.attr("qualified_name"); + boost::python::object visibility = pyClass.attr("visibility"); boost::optional fileLoc = createFileLocFromPythonFilePosition(pyClass.attr("file_position")); @@ -319,6 +325,7 @@ void Persistence::persistClass(boost::python::object pyClass) cl->astNodeId = classAstNode->id; cl->name = boost::python::extract(name); cl->qualifiedName = boost::python::extract(qualifiedName); + cl->visibility = boost::python::extract(visibility); cl->id = model::creatIdentifier(*cl); diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 585f12fc3..8063dd7d4 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -3,6 +3,8 @@ #include +#include + #include #include @@ -233,6 +235,8 @@ class PythonServiceHandler : virtual public LanguageServiceIf model::PythonEntity queryPythonEntityByAstNode(const model::AstNodeId& id); + std::map getVisibilities(const std::vector& nodes_); + std::shared_ptr _db; util::OdbTransaction _transaction; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 5dbdffc07..312324bd2 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -45,27 +45,43 @@ namespace typedef odb::query FileQuery; typedef odb::result FileResult; - cc::service::language::AstNodeInfo createAstNodeInfo(const cc::model::PythonAstNode& astNode_) + struct CreateAstNodeInfo { - cc::service::language::AstNodeInfo ret; + typedef std::map VisibilityMap; - ret.__set_id(std::to_string(astNode_.id)); - ret.__set_entityHash(0); - ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); - ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); - ret.__set_astNodeValue(astNode_.astValue); + CreateAstNodeInfo(const VisibilityMap& visibilities_ = {}) : _visibilities(visibilities_) + { + } + + cc::service::language::AstNodeInfo operator()(const cc::model::PythonAstNode& astNode_) + { + cc::service::language::AstNodeInfo ret; + + ret.__set_id(std::to_string(astNode_.id)); + ret.__set_entityHash(0); + ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); + ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); + ret.__set_astNodeValue(astNode_.astValue); + + VisibilityMap::const_iterator it = _visibilities.find(astNode_.id); + if (it != _visibilities.end()){ + ret.__set_tags({it->second}); + } + + ret.range.range.startpos.line = astNode_.location.range.start.line; + ret.range.range.startpos.column = astNode_.location.range.start.column; + ret.range.range.endpos.line = astNode_.location.range.end.line; + ret.range.range.endpos.column = astNode_.location.range.end.column; - ret.range.range.startpos.line = astNode_.location.range.start.line; - ret.range.range.startpos.column = astNode_.location.range.start.column; - ret.range.range.endpos.line = astNode_.location.range.end.line; - ret.range.range.endpos.column = astNode_.location.range.end.column; + if (astNode_.location.file){ + ret.range.file = std::to_string(astNode_.location.file.object_id()); + } - if (astNode_.location.file){ - ret.range.file = std::to_string(astNode_.location.file.object_id()); + return ret; } - return ret; - } + const VisibilityMap& _visibilities; + }; } namespace cc { @@ -94,7 +110,7 @@ void PythonServiceHandler::getAstNodeInfo( const core::AstNodeId& astNodeId_) { return_ = _transaction([this, &astNodeId_](){ - return createAstNodeInfo(queryPythonAstNode(astNodeId_)); + return CreateAstNodeInfo()(queryPythonAstNode(astNodeId_)); }); } @@ -131,7 +147,7 @@ void PythonServiceHandler::getAstNodeInfoByPosition( } return_ = _transaction([this, &min](){ - return createAstNodeInfo(min); + return CreateAstNodeInfo(getVisibilities({min}))(min); }); }); } @@ -218,18 +234,18 @@ void PythonServiceHandler::getProperties( break; } -// case model::PythonAstNode::SymbolType::Module: -// { -// ModImpResult modules = _db->query( -// ModImpQuery::astNodeId == node.id); -// model::PythonImport module = *modules.begin(); -// -// return_["From"] = module.imported->filename; -// return_["To"] = module.importer->filename; -// //return_["Symbol"] = imported symbol -// -// break; -// } + case model::PythonAstNode::SymbolType::Module: + { + ModImpResult modules = _db->query( + ModImpQuery::astNodeId == node.id); + model::PythonImport module = *modules.begin(); + + return_["From"] = module.imported->filename; + //return_["To"] = module.importer->filename; + //return_["Symbol"] = imported symbol + + break; + } } }); } @@ -561,7 +577,7 @@ void PythonServiceHandler::getReferences( std::transform( nodes.begin(), nodes.end(), std::back_inserter(return_), - createAstNodeInfo); + CreateAstNodeInfo(getVisibilities(nodes))); }); }); } @@ -767,7 +783,7 @@ void PythonServiceHandler::getFileReferences( std::transform( nodes.begin(), nodes.end(), std::back_inserter(return_), - createAstNodeInfo); + CreateAstNodeInfo(getVisibilities(nodes))); }); }); } @@ -1029,6 +1045,26 @@ model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const model return *entities.begin(); } +std::map PythonServiceHandler::getVisibilities( + const std::vector& nodes_) +{ + std::map visibilities; + + for (const model::PythonAstNode& node : nodes_) + { + switch(node.symbolType){ + case mode::PythonAstNode::SymbolType::Variable: + case mode::PythonAstNode::SymbolType::Function: + case mode::PythonAstNode::SymbolType::Class: + model::PythonEntity entity = queryPythonEntity(node); + visibilities[node.id] = entity.visibility; + break; + } + } + + return visibilities; +} + } } } \ No newline at end of file From 9d384f216a25ff24ed8dd0efe84a31d587def912 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Wed, 11 Nov 2020 20:54:28 +0100 Subject: [PATCH 06/39] Get entity by qualified name. Set AstNodeInfo entity hash in service. --- .../model/include/model/pythonastnode.h | 5 +++ plugins/python/parser/src/pythonparser.cpp | 43 ++++++------------- plugins/python/service/src/pythonservice.cpp | 4 +- 3 files changed, 19 insertions(+), 33 deletions(-) diff --git a/plugins/python/model/include/model/pythonastnode.h b/plugins/python/model/include/model/pythonastnode.h index 3347c7c27..d34e84f29 100644 --- a/plugins/python/model/include/model/pythonastnode.h +++ b/plugins/python/model/include/model/pythonastnode.h @@ -134,6 +134,11 @@ inline std::uint64_t createIdentifier(const PythonAstNode& astNode_) return util::fnvHash(res); } +inline std::uint64_t createAstNodeInfoEntityHash(const PythonAstNode& astNode_) +{ + return util::fnvHash(astNode_.qualifiedName); +} + #pragma db view \ object(PythonAstNode) object(File = LocFile : PythonAstNode::location.file) \ query ((?) + "GROUP BY" + LocFile::id + "ORDER BY" + LocFile::id) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index f718cce0b..fa850f4f5 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -57,7 +57,7 @@ class Persistence private: boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); - model::PythonEntity* getPythonEntity(const std::string& qualifiedName, const model::FileLoc& fileLoc); + model::PythonEntity* getPythonEntity(const std::string& qualifiedName); }; void Persistence::persistFile(boost::python::object pyFile) @@ -165,7 +165,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); + type->type = getPythonEntity(boost::python::extract(types[i]))->id; type->symbol = variable->id; transaction([&, this] { @@ -254,7 +254,7 @@ void Persistence::persistFunction(boost::python::object pyFunction) for(int i = 0; itype = getPythonEntityIdByQualifiedName(boost::python::extract(types[i])); + type->type = getPythonEntity(boost::python::extract(types[i]))->id; type->symbol = function->id; transaction([&, this] { @@ -362,7 +362,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iderived = cl->id; - // inheritance->base = getPythonEntityIdByQualifiedName(boost::python::extract(baseClasses[i])); + inheritance->base = getPythonEntity(boost::python::extract(baseClasses[i]))->id; transaction([&, this] { ctx.db->persist(inheritance); @@ -383,7 +383,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - // classMember->memberId = search for function declaration + classMember->memberId = getPythonEntity(boost::python::extract(methods[i]))->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; @@ -396,7 +396,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - // classMember->memberId = search for function declaration + classMember->memberId = getPythonEntity(boost::python::extract(staticMethods[i]))->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = true; @@ -409,7 +409,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - // classMember->memberId = search for variable declaration + classMember->memberId = getPythonEntity(boost::python::extract(attributes[i]))->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = false; @@ -422,7 +422,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - // classMember->memberId = search for variable declaration + classMember->memberId = getPythonEntity(boost::python::extract(staticAttributes[i]))->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = true; @@ -435,7 +435,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - // classMember->memberId = search for class declaration + classMember->memberId = getPythonEntity(boost::python::extract(classes[i]))->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Class; classMember->staticMember = false; @@ -491,7 +491,7 @@ void Persistence::persistImport(boost::python::object pyImport) model::PythonImportPtr moduleImport(new model::PythonImport); moduleImport->importer = file; moduleImport->imported = moduleFile; -// moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1])); + moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1]))->id; transaction([&, this] { ctx.db->persist(moduleImport); @@ -536,32 +536,13 @@ boost::optional Persistence::createFileLocFromPythonFilePosition return fileLoc; } -model::PythonEntity* Persistence::getPythonEntity(const std::string& qualifiedName, const model::FileLoc& fileLoc) +model::PythonEntity* Persistence::getPythonEntity(const std::string& qualifiedName) { - using odb::query AstQuery; - using odb::result AstResult; using odb::query EntityQuery; using odb::result EntityResult; - const model::Position& start = astNode_.location.range.start; - const model::Position& end = astNode_.location.range.end; - - AstResult astNode = ctx.db->query( - AstQuery::location.file == astNode_.location.file.object_id() && - // StartPos == Pos - (AstQuery::location.range.start.line == start.line && - AstQuery::location.range.start.column == start.column) && - // Pos == EndPos - (AstQuery::location.range.end.line == end.line && - AstQuery::location.range.end.column == end.column)); - - if (astNode.empty()){ - return nullptr; - } - EntityResult entity = ctx.db->query( - EntityQuery::qualifiedName == qualifiedName && - EntityQuery::astNodeId == astNode.begin()->id); + EntityQuery::qualifiedName == qualifiedName); if (entity.empty()){ return nullptr; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 312324bd2..f6d7692a1 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -58,7 +58,7 @@ namespace cc::service::language::AstNodeInfo ret; ret.__set_id(std::to_string(astNode_.id)); - ret.__set_entityHash(0); + ret.__set_entityHash(cc::model::createAstNodeInfoEntityHash(astNode_)); ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); ret.__set_astNodeValue(astNode_.astValue); @@ -217,7 +217,7 @@ void PythonServiceHandler::getProperties( return_["Name"] = function.qualifiedName.substr( function.qualifiedName.find_last_of('.') + 1); return_["Qualified name"] = function.qualifiedName; - return_["Signature"] = function.name; + return_["Signature"] = function.name; // TODO break; } From 41f1cc018837b6c6125c31236905e98e532dab95 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Wed, 11 Nov 2020 21:39:25 +0100 Subject: [PATCH 07/39] Python plugin webgui scripts --- plugins/python/webgui/js/pythonInfoTree.js | 250 ++++++++++++++++++++- plugins/python/webgui/js/pythonMenu.js | 67 ++++++ 2 files changed, 316 insertions(+), 1 deletion(-) diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index f2eb1c797..c4b289f6f 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -5,7 +5,255 @@ require([ function (model, viewHandler, util){ model.addService('pythonservice', 'PythonService', LanguageServiceClient); - var pythonInfoTree = {}; + function createLabel(astNodeInfo) { + var labelValue = astNodeInfo.astNodeValue; + + var label = astNodeInfo.range.range.startpos.line + ':' + + astNodeInfo.range.range.startpos.column + ': ' + + labelValue + + ''; + } + + function getCssClass(astNodeInfo) { + var tags = astNodeInfo.tags; + + return tags.indexOf('public') > -1 ? 'icon-visibility icon-public' : + tags.indexOf('private') > -1 ? 'icon-visibility icon-private' : + tags.indexOf('semiprivate') > -1 ? 'icon-visibility icon-protected' : + null; + } + + function groupReferencesByVisibilities(references, parentNode, nodeInfo){ + var res = []; + var visibilities = ['public', 'semiprivate', 'private'] + + visibilities.forEach(function (visibility){ + var nodes = references.filter(function (reference){ + return reference.tags.indexOf(visibility) > -1; + }); + + if (nodes.length === 0){ + return; + } + + var visibility_icon_name = visibility === 'semiprivate' ? 'protected' : visibility; + + res.push({ + id: nodeInfo.id + visibility + parentNode.refType, + name: createReferenceCountLabel(visibility, nodes.length), + refType: parentNode.refType, + hasChildren: true, + cssClass: 'icon-visibility icon-' + visibility_icon_name, + getChildren: function (){ + var res = []; + nodes.forEach(function (reference){ + res.push({ + id: visibility + reference.id, + name: createLabel(reference), + refType: parentNode.refType, + nodeInfo: reference, + hasChildren: false, + cssClass: getCssClass(reference) + }); + }); + return res; + } + }); + }); + + return res; + } + + function createRootNode(elementInfo) { + var rootLabel + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.symbolType + : 'File') + + ''; + + var rootValue + = '' + + (elementInfo instanceof AstNodeInfo + ? elementInfo.astNodeValue + : elementInfo.name) + + ''; + + var label = createTagLabels(elementInfo.tags) // no tags in python + + '' + + rootLabel + ': ' + rootValue + + ''; + + return { + id : 'root', + name : label, + cssClass : 'icon-info', + hasChildren : true, + getChildren : function () { + return this._store.query({ parent : 'root' }); + } + }; + } + + function createReferenceCountLabel(label, count) { + return label + '(' + count + ')'; + } + + function loadReferenceNodes(parentNode, nodeInfo, refTypes) { + var res = []; + var fileGroupsId = []; + + var references = model.pythonservice.getReferences(nodeInfo.id, parentNode.refType); + + if (parentNode.refType === refTypes['Method'] || + parentNode.refType === refTypes['Data member']){ + return groupReferencesByVisibilities(references, parentNode, nodeInfo); + } + + references.forEach(function (reference) { + if (parentNode.refType === refTypes['Caller'] || + parentNode.refType === refTypes['Usage']) { + var fileId = reference.range.file; + if (fileGroupsId[fileId]){ + return; + } + + fileGroupsId[fileId] = parentNode.refType + fileId + reference.id; + + var referenceInFile = references.filter(function (ref) { + return reference.range.file === fileId; + }); + + var fileInfo = model.project.getFileInfo(fileId); + res.push({ + id: fileGroupsId[fileId], + name: createReferenceCountLabel(fileInfo.name, referenceInFile.length), + refType: parentNode.refType, + hasChildren: true, + cssClass: util.getIconClass(fileInfo.path), + getChildren: function () { + var that = this; + var res = []; + + referenceInFile.forEach(function (ref) { + if (parentNode.refType === refTypes['Caller']) { + ref.push({ + id: ref.id, + name: createLabel(ref), + nodeInfo: ref, + refType: parentNode.refType, + cssClass: 'icon icon-Method', + hasChildren: true, + getChildren: function () { + var res = []; + + var refCount = model.pythonservice.getReferenceCount(ref.id, parentNode.refType); + + if (refCount > 0) { + res.push({ + id: 'Caller-' + ref.id, + name: createReferenceCountLabel(parentNode.name, refCount), + nodeInfo: ref, + refType: parentNode.refType, + cssClass: parentNode.cssClass, + hasChildren: true, + getChildren: parentNode.getChildren + }); + } + + var calls = model.pythonservice.getReferences(this.nodeInfo.id, refTypes['This calls']); + + calls.forEach(function (call) { + if (call.entityHash === nodeInfo.entityHash) + res.push({ + name: createLabel(call), + refType: parentNode.refType, + nodeInfo: call, + hasChildren: false, + cssClass: getCssClass(call) + }); + }); + + return res; + } + }); + } else if (parentNode.refType === refTypes['Usage']) { + res.push({ + id: fileGroupsId[fileId] + ref.id, + name: createLabel(ref), + refType: parentNode.refType, + nodeInfo: ref, + hasChildren: false, + cssClass: getCssClass(ref) + }); + } + }); + return res; + } + }); + } else { + res.push({ + name: createLabel(reference), + refType: parentNode.refType, + nodeInfo: reference, + hasChildren: false, + cssClass: getCssClass(reference) + }); + } + }); + + return res; + } + + var pythonInfoTree = { + render : function (elementInfo){ + var ret = []; + + ret.push(createRootNode(elementInfo)); + + if (elementInfo instanceof AstNodeInfo){ + var properties = model.pythonservice.getProperties(elementInfo.id); + + for (var propertyName in properties) { + var propertyId = propertyName.replace(/ /g, '-'); + var label = + '' + propertyName + '' + + '' + properties[propertyName] +''; + + ret.push({ + name: label, + parent: 'root', + nodeInfo: elementInfo, + cssClass: 'icon-' + propertyId, + hasChildren: false + }); + } + + var refTypes = model.pythonservice.getReferenceTypes(elementInfo.id); + + for (var refType in refTypes) { + var refCount = model.pythonservice.getReferenceCount(elementInfo.id, refTypes[refType]); + + if (refCount > 0){ + ret.push({ + name: createReferenceCountLabel(refType, refCount), + parent: 'root', + refType: refTypes[refType], + cssClass: 'icon-' + refType.replace(/ /g, '-'), + hasChildren: true, + getChildren: function () { + return loadReferenceNodes(this, elementInfo, refTypes); + } + }); + } + } + } else if (elementInfo instanceof FileInfo) { + + } + + return ret; + } + }; viewHandler.registerModule(pythonInfoTree, { type : viewHandler.moduleType.InfoTree, diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js index ebf31c5fc..4595d0303 100644 --- a/plugins/python/webgui/js/pythonMenu.js +++ b/plugins/python/webgui/js/pythonMenu.js @@ -8,4 +8,71 @@ require([ 'codecompass/viewHandler'], function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, viewHandler){ model.addService('pythonservice', 'PythonService', LanguageServiceClient); + + var getdefintion = { + id : 'python-text-getdefintion', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Jump to definition', + accelKey : 'ctrl - click', + onClick : function () { + if (!nodeInfo || !fileInfo) { + return; + } + + // var languageService = model.getLanguageService(fileInfo.type); + astHelper.jumpToDef(nodeInfo.id, model.pythonservice); + } + }); + } + }; + + viewHandler.registerModule(getdefintion, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infoTree = { + id : 'python-text-infotree', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Info Tree', + onClick : function () { + if (!nodeInfo || !fileInfo) { + return; + } + + topic.publish('codecompass/infotree', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + } + }); + } + }; + + viewHandler.registerModule(infoTree, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); + + var infobox = { + id : 'python-text-infobox', + render : function (nodeInfo, fileInfo) { + return new MenuItem({ + label : 'Documentation', + onClick : function () { + topic.publish('codecompass/documentation', { + fileType : fileInfo.type, + elementInfo : nodeInfo + }); + } + }); + } + }; + + viewHandler.registerModule(infobox, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); }); \ No newline at end of file From f4cc31c9070021651df38c2f01ba6bf6a444abc2 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sat, 14 Nov 2020 18:49:29 +0100 Subject: [PATCH 08/39] Fix compilation --- .../model/include/model/pythonastnode.h | 12 +-- .../python/model/include/model/pythonimport.h | 5 ++ plugins/python/parser/CMakeLists.txt | 3 +- plugins/python/parser/src/pythonparser.cpp | 85 +++++++++++++------ .../service/include/service/pythonservice.h | 2 +- plugins/python/service/src/pythonservice.cpp | 19 +++-- 6 files changed, 84 insertions(+), 42 deletions(-) diff --git a/plugins/python/model/include/model/pythonastnode.h b/plugins/python/model/include/model/pythonastnode.h index d34e84f29..0c91bd441 100644 --- a/plugins/python/model/include/model/pythonastnode.h +++ b/plugins/python/model/include/model/pythonastnode.h @@ -116,17 +116,17 @@ inline std::uint64_t createIdentifier(const PythonAstNode& astNode_) res .append(astNode_.astValue).append(":") - .append(std::to_string(astNode_.qualifiedName)).append(":") + .append(astNode_.qualifiedName).append(":") .append(std::to_string(static_cast(astNode_.symbolType))).append(":") .append(std::to_string(static_cast(astNode_.astType))).append(":"); - if (astNode_.location.file != nullptr){ + if (astNode_.location.file){ res .append(std::to_string(astNode_.location.file->id)).append(":") - .append(std::to_string(astNode_.location.file->range.start.line)).append(":") - .append(std::to_string(astNode_.location.file->range.start.column)).append(":") - .append(std::to_string(astNode_.location.file->range.end.line)).append(":") - .append(std::to_string(astNode_.location.file->range.end.column)).append(":"); + .append(std::to_string(astNode_.location.range.start.line)).append(":") + .append(std::to_string(astNode_.location.range.start.column)).append(":") + .append(std::to_string(astNode_.location.range.end.line)).append(":") + .append(std::to_string(astNode_.location.range.end.column)).append(":"); } else { res.append("null"); } diff --git a/plugins/python/model/include/model/pythonimport.h b/plugins/python/model/include/model/pythonimport.h index 7bf792b7a..bd47b4c91 100644 --- a/plugins/python/model/include/model/pythonimport.h +++ b/plugins/python/model/include/model/pythonimport.h @@ -5,6 +5,8 @@ #include +#include + #include "pythonentity.h" namespace cc @@ -18,6 +20,9 @@ struct PythonImport #pragma db id auto int id; + #pragma db unique + PythonAstNodeId astNodeId; + #pragma db not_null #pragma db on_delete(cascade) odb::lazy_shared_ptr importer; diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 0e98dd3f0..62291ce69 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -10,8 +10,7 @@ include_directories( ${PLUGIN_DIR}/model/include) add_library(pythonparser SHARED - src/pythonparser.cpp - src/a/asdf.py) + src/pythonparser.cpp) find_package (Python) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index fa850f4f5..290c6c128 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -36,6 +36,7 @@ #include #include #include +#include namespace cc { namespace parser{ @@ -57,7 +58,7 @@ class Persistence private: boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); - model::PythonEntity* getPythonEntity(const std::string& qualifiedName); + boost::optional getPythonEntity(const std::string& qualifiedName); }; void Persistence::persistFile(boost::python::object pyFile) @@ -165,7 +166,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) for(int i = 0; itype = getPythonEntity(boost::python::extract(types[i]))->id; + type->type = getPythonEntity(boost::python::extract(types[i])).get().id; type->symbol = variable->id; transaction([&, this] { @@ -254,7 +255,7 @@ void Persistence::persistFunction(boost::python::object pyFunction) for(int i = 0; itype = getPythonEntity(boost::python::extract(types[i]))->id; + type->type = getPythonEntity(boost::python::extract(types[i])).get().id; type->symbol = function->id; transaction([&, this] { @@ -327,7 +328,7 @@ void Persistence::persistClass(boost::python::object pyClass) cl->qualifiedName = boost::python::extract(qualifiedName); cl->visibility = boost::python::extract(visibility); - cl->id = model::creatIdentifier(*cl); + cl->id = model::createIdentifier(*cl); transaction([&, this] { ctx.db->persist(cl); @@ -362,7 +363,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iderived = cl->id; - inheritance->base = getPythonEntity(boost::python::extract(baseClasses[i]))->id; + inheritance->base = getPythonEntity(boost::python::extract(baseClasses[i])).get().id; transaction([&, this] { ctx.db->persist(inheritance); @@ -383,7 +384,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(methods[i]))->id; + classMember->memberId = getPythonEntity(boost::python::extract(methods[i])).get().id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; @@ -396,7 +397,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(staticMethods[i]))->id; + classMember->memberId = getPythonEntity(boost::python::extract(staticMethods[i])).get().id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = true; @@ -409,7 +410,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(attributes[i]))->id; + classMember->memberId = getPythonEntity(boost::python::extract(attributes[i])).get().id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = false; @@ -422,7 +423,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(staticAttributes[i]))->id; + classMember->memberId = getPythonEntity(boost::python::extract(staticAttributes[i])).get().id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = true; @@ -435,7 +436,7 @@ void Persistence::persistClass(boost::python::object pyClass) for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(classes[i]))->id; + classMember->memberId = getPythonEntity(boost::python::extract(classes[i])).get().id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Class; classMember->staticMember = false; @@ -468,15 +469,30 @@ void Persistence::persistImport(boost::python::object pyImport) util::OdbTransaction transaction{ctx.db}; for (int i = 0; i < boost::python::len(importedModules); ++i) { - model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importedModules[i])); - if (moduleFile == nullptr) { + boost::python::object importData = importedModules[i]; + + model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); + boost::optional fileLoc = createFileLocFromPythonFilePosition(importData.attr("position")); + + if (moduleFile == nullptr || fileLoc == boost::none) { continue; } + + model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); + moduleAstNode->location = fileLoc.get(); + moduleAstNode->qualifiedName = ""; + moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; + moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; + + moduleAstNode->id = model::createIdentifier(*moduleAstNode); + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; transaction([&, this] { + ctx.db->persist(moduleAstNode); ctx.db->persist(moduleImport); }); } @@ -484,17 +500,38 @@ void Persistence::persistImport(boost::python::object pyImport) boost::python::list importDict = importedSymbols.items(); for (int i = 0; i < boost::python::len(importDict); ++i) { boost::python::tuple import = boost::python::extract(importDict[i]); - model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(import[0])); - if (moduleFile == nullptr) { + + boost::python::object importData = import[0]; + + model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); + boost::optional fileLoc = createFileLocFromPythonFilePosition(importData.attr("position")); + + if (moduleFile == nullptr || fileLoc == boost::none) { continue; } - model::PythonImportPtr moduleImport(new model::PythonImport); - moduleImport->importer = file; - moduleImport->imported = moduleFile; - moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1]))->id; + + model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); + moduleAstNode->location = fileLoc.get(); + moduleAstNode->qualifiedName = ""; + moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; + moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; + + moduleAstNode->id = model::createIdentifier(*moduleAstNode); + + for (int j = 0; j < boost::python::len(import[1]); ++j){ + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->astNodeId = moduleAstNode->id; + moduleImport->importer = file; + moduleImport->imported = moduleFile; + moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1][j])).get().id; + + transaction([&, this] { + ctx.db->persist(moduleImport); + }); + } transaction([&, this] { - ctx.db->persist(moduleImport); + ctx.db->persist(moduleAstNode); }); } } catch(std::exception e){ @@ -536,19 +573,19 @@ boost::optional Persistence::createFileLocFromPythonFilePosition return fileLoc; } -model::PythonEntity* Persistence::getPythonEntity(const std::string& qualifiedName) +boost::optional Persistence::getPythonEntity(const std::string& qualifiedName) { - using odb::query EntityQuery; - using odb::result EntityResult; + using EntityQuery = odb::query; + using EntityResult = odb::result; EntityResult entity = ctx.db->query( EntityQuery::qualifiedName == qualifiedName); if (entity.empty()){ - return nullptr; + return boost::none; } - return entity.begin(); + return *entity.begin(); } typedef boost::shared_ptr PersistencePtr; diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 8063dd7d4..05ca1d999 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -233,7 +233,7 @@ class PythonServiceHandler : virtual public LanguageServiceIf model::PythonEntity queryPythonEntity(const model::PythonEntityId& id); - model::PythonEntity queryPythonEntityByAstNode(const model::AstNodeId& id); + model::PythonEntity queryPythonEntityByAstNode(const model::PythonAstNodeId& id); std::map getVisibilities(const std::vector& nodes_); diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index f6d7692a1..9ff7b7564 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -176,16 +176,17 @@ void PythonServiceHandler::getDocumentation( std::string& return_, const core::AstNodeId& astNodeId_) { - PythonEntity entity = queryPythonEntityByAstNode(astNodeId_); + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + model::PythonEntity entity = queryPythonEntityByAstNode(node.id); DocResult doc = _db->query( DocQuery::documented == entity.id); if (doc.empty()){ - return std::string(); + return_ = std::string(); + } else { + return_ = doc.begin()->documentation; } - - return_ = doc.begin()->documentation; } void PythonServiceHandler::getProperties( @@ -1039,7 +1040,7 @@ model::PythonEntity PythonServiceHandler::queryPythonEntity(const model::PythonE return *entities.begin(); } -model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const model::AstNodeId& id) +model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const model::PythonAstNodeId& id) { EntityResult entities = _db->query(EntityQuery::astNodeId == id); return *entities.begin(); @@ -1053,10 +1054,10 @@ std::map PythonServiceHandler::getVisibilit for (const model::PythonAstNode& node : nodes_) { switch(node.symbolType){ - case mode::PythonAstNode::SymbolType::Variable: - case mode::PythonAstNode::SymbolType::Function: - case mode::PythonAstNode::SymbolType::Class: - model::PythonEntity entity = queryPythonEntity(node); + case model::PythonAstNode::SymbolType::Variable: + case model::PythonAstNode::SymbolType::Function: + case model::PythonAstNode::SymbolType::Class: + model::PythonEntity entity = queryPythonEntityByAstNode(node.id); visibilities[node.id] = entity.visibility; break; } From 8b7a3bdcf19015027e23236a023c19881cb51ecc Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Tue, 24 Nov 2020 21:25:37 +0100 Subject: [PATCH 09/39] Fix Persistence, and PythonService usage during web server --- plugins/cpp/parser/src/cppparser.cpp | 2 + .../python/model/include/model/pythontype.h | 8 + .../include/pythonparser/pythonparser.h | 23 +- plugins/python/parser/src/pythonparser.cpp | 289 +++++++++++------- plugins/python/service/CMakeLists.txt | 3 +- plugins/python/service/src/plugin.cpp | 26 ++ 6 files changed, 236 insertions(+), 115 deletions(-) create mode 100644 plugins/python/service/src/plugin.cpp diff --git a/plugins/cpp/parser/src/cppparser.cpp b/plugins/cpp/parser/src/cppparser.cpp index c7a737234..98cc68d08 100644 --- a/plugins/cpp/parser/src/cppparser.cpp +++ b/plugins/cpp/parser/src/cppparser.cpp @@ -680,6 +680,8 @@ bool CppParser::cleanupWorker(const std::string& path_) bool CppParser::parse() { + return true; + initBuildActions(); VisitorActionFactory::init(_ctx); diff --git a/plugins/python/model/include/model/pythontype.h b/plugins/python/model/include/model/pythontype.h index f8dcfeda0..e9e598874 100644 --- a/plugins/python/model/include/model/pythontype.h +++ b/plugins/python/model/include/model/pythontype.h @@ -16,6 +16,14 @@ struct PythonType PythonEntityId type; PythonEntityId symbol; + + std::string toString() const + { + return std::string("PythonType") + .append("\nid = ").append(std::to_string(id)) + .append("\ntype = ").append(std::to_string(type)) + .append("\nsymbol = ").append(std::to_string(symbol)); + } }; typedef std::shared_ptr PythonTypePtr; diff --git a/plugins/python/parser/include/pythonparser/pythonparser.h b/plugins/python/parser/include/pythonparser/pythonparser.h index 877b7bab6..d2b36fafb 100644 --- a/plugins/python/parser/include/pythonparser/pythonparser.h +++ b/plugins/python/parser/include/pythonparser/pythonparser.h @@ -7,22 +7,23 @@ namespace cc { - namespace parser - { +namespace parser +{ + +class PythonParser : public AbstractParser { +public: + PythonParser(ParserContext &ctx_); - class PythonParser : public AbstractParser { - public: - PythonParser(ParserContext &ctx_); + virtual ~PythonParser(); - virtual ~PythonParser(); + virtual void markModifiedFiles() override; - virtual void markModifiedFiles() override; + virtual bool cleanupDatabase() override; - virtual bool cleanupDatabase() override; + virtual bool parse() override; +}; - virtual bool parse() override; - }; - } // parser +} // parser } // cc #endif //CODECOMPASS_PYTHONPARSER_H diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 290c6c128..0ebfe50d5 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -41,6 +41,10 @@ namespace cc { namespace parser{ +void print(const std::string& s) { + std::cout << s << std::endl; +} + class Persistence { private: @@ -48,8 +52,10 @@ class Persistence public: Persistence(ParserContext& ctx_) : ctx(ctx_) {} + ~Persistence(); void f() { std::cout << "C++" << std::endl; } + void g(boost::python::object o); void persistFile(boost::python::object pyFile); void persistVariable(boost::python::object pyVariable); void persistFunction(boost::python::object pyFunction); @@ -59,12 +65,43 @@ class Persistence private: boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); boost::optional getPythonEntity(const std::string& qualifiedName); + + std::vector _astNodes; + std::vector _variables; + std::vector _functions; + std::vector _classes; + std::vector _members; + std::vector _inheritance; + std::vector _imports; + std::vector _documentations; + std::vector _types; }; +Persistence::~Persistence() +{ + ctx.srcMgr.persistFiles(); + + (util::OdbTransaction(ctx.db))([this]{ + util::persistAll(_astNodes, ctx.db); + util::persistAll(_variables, ctx.db); + util::persistAll(_functions, ctx.db); + util::persistAll(_classes, ctx.db); + util::persistAll(_members, ctx.db); + util::persistAll(_inheritance, ctx.db); + util::persistAll(_imports, ctx.db); + util::persistAll(_documentations, ctx.db); + util::persistAll(_types, ctx.db); + }); +} + +void Persistence::g(boost::python::object o) { + std::string s = boost::python::extract(o); + std::cout << s << std::endl; +} + void Persistence::persistFile(boost::python::object pyFile) { try{ - std::cout << "0" << std::endl; model::FilePtr file = nullptr; model::BuildSource buildSource; @@ -121,14 +158,12 @@ void Persistence::persistFile(boost::python::object pyFile) void Persistence::persistVariable(boost::python::object pyVariable) { try{ - std::cout << "1" << std::endl; boost::python::object name = pyVariable.attr("name"); boost::python::object qualifiedName = pyVariable.attr("qualified_name"); boost::python::object visibility = pyVariable.attr("visibility"); boost::optional fileLoc = createFileLocFromPythonFilePosition(pyVariable.attr("file_position")); - // set boost::python::list types = boost::python::extract(pyVariable.attr("type")); boost::python::list usages = boost::python::extract(pyVariable.attr("usages")); @@ -147,11 +182,10 @@ void Persistence::persistVariable(boost::python::object pyVariable) varAstNode->astType = model::PythonAstNode::AstType::Declaration; varAstNode->id = model::createIdentifier(*varAstNode); - - transaction([&, this] { - ctx.db->persist(varAstNode); - }); - + // transaction([&, this] { + // ctx.db->persist(*varAstNode); + // }); + _astNodes.push_back(varAstNode); model::PythonVariablePtr variable(new model::PythonVariable); variable->astNodeId = varAstNode->id; variable->name = boost::python::extract(name); @@ -159,19 +193,18 @@ void Persistence::persistVariable(boost::python::object pyVariable) variable->visibility = boost::python::extract(visibility); variable->id = model::createIdentifier(*variable); - - transaction([&, this] { - ctx.db->persist(variable); - }); - + // transaction([&, this] { + // ctx.db->persist(*variable); + // }); + _variables.push_back(variable); for(int i = 0; itype = getPythonEntity(boost::python::extract(types[i])).get().id; type->symbol = variable->id; - - transaction([&, this] { - ctx.db->persist(type); - }); + // transaction([&, this] { + // ctx.db->persist(type); + // }); + _types.push_back(type); } for(int i = 0; isymbolType = model::PythonAstNode::SymbolType::Variable; usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(usageAstNode); - }); + // transaction([&, this] { + // ctx.db->persist(usageAstNode); + // }); + _astNodes.push_back(usageAstNode); } - } catch(std::exception e){ - std::cout << e.what() << std::endl; + + } catch (const odb::object_already_persistent& ex) + { + std::cout << "Var exception already persistent:" << std::endl; + std::cout << ex.what() << std::endl; + } catch (const odb::database_exception& ex) + { + std::cout << "Var exception db exception:" << std::endl; + std::cout << ex.what() << std::endl; + } catch(std::exception ex){ + std::cout << "Var exception:" << std::endl; + std::cout << ex.what() << std::endl; } } void Persistence::persistFunction(boost::python::object pyFunction) { try{ - std::cout << "2" << std::endl; boost::python::object name = pyFunction.attr("name"); boost::python::object qualifiedName = pyFunction.attr("qualified_name"); boost::python::object visibility = pyFunction.attr("visibility"); - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyFunction.attr("file_position")); - // set boost::python::list types = boost::python::extract(pyFunction.attr("type")); boost::python::list usages = boost::python::extract(pyFunction.attr("usages")); @@ -228,9 +269,10 @@ void Persistence::persistFunction(boost::python::object pyFunction) funcAstNode->id = model::createIdentifier(*funcAstNode); - transaction([&, this] { - ctx.db->persist(funcAstNode); - }); + // transaction([&, this] { + // ctx.db->persist(funcAstNode); + // }); + _astNodes.push_back(funcAstNode); model::PythonFunctionPtr function(new model::PythonFunction); function->astNodeId = funcAstNode->id; @@ -240,27 +282,30 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->id = model::createIdentifier(*function); - transaction([&, this] { - ctx.db->persist(function); - }); + // transaction([&, this] { + // ctx.db->persist(function); + // }); + _functions.push_back(function); model::PythonDocumentationPtr documentation(new model::PythonDocumentation); documentation->documentation = boost::python::extract(pyDocumentation); documentation->documented = function->id; documentation->documentationKind = model::PythonDocumentation::Function; - transaction([&, this] { - ctx.db->persist(documentation); - }); + // transaction([&, this] { + // ctx.db->persist(documentation); + // }); + _documentations.push_back(documentation); for(int i = 0; itype = getPythonEntity(boost::python::extract(types[i])).get().id; type->symbol = function->id; - transaction([&, this] { - ctx.db->persist(type); - }); + // transaction([&, this] { + // ctx.db->persist(type); + // }); + _types.push_back(type); } for(int i = 0; isymbolType = model::PythonAstNode::SymbolType::Function; usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(usageAstNode); - }); + // transaction([&, this] { + // ctx.db->persist(usageAstNode); + // }); + _astNodes.push_back(usageAstNode); } } catch(std::exception e){ + std::cout << "Func exception:" << std::endl; std::cout << e.what() << std::endl; } } @@ -287,7 +334,6 @@ void Persistence::persistFunction(boost::python::object pyFunction) void Persistence::persistClass(boost::python::object pyClass) { try{ - std::cout << "3" << std::endl; boost::python::object name = pyClass.attr("name"); boost::python::object qualifiedName = pyClass.attr("qualified_name"); boost::python::object visibility = pyClass.attr("visibility"); @@ -298,7 +344,6 @@ void Persistence::persistClass(boost::python::object pyClass) boost::python::object pyDocumentation = pyClass.attr("documentation"); - // set boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); boost::python::object members = pyClass.attr("members"); @@ -318,9 +363,10 @@ void Persistence::persistClass(boost::python::object pyClass) classAstNode->id = model::createIdentifier(*classAstNode); - transaction([&, this] { - ctx.db->persist(classAstNode); - }); + // transaction([&, this] { + // ctx.db->persist(classAstNode); + // }); + _astNodes.push_back(classAstNode); model::PythonClassPtr cl(new model::PythonClass); cl->astNodeId = classAstNode->id; @@ -330,18 +376,20 @@ void Persistence::persistClass(boost::python::object pyClass) cl->id = model::createIdentifier(*cl); - transaction([&, this] { - ctx.db->persist(cl); - }); + // transaction([&, this] { + // ctx.db->persist(cl); + // }); + _classes.push_back(cl); model::PythonDocumentationPtr documentation(new model::PythonDocumentation); documentation->documentation = boost::python::extract(pyDocumentation); documentation->documented = cl->id; documentation->documentationKind = model::PythonDocumentation::Class; - transaction([&, this] { - ctx.db->persist(documentation); - }); + // transaction([&, this] { + // ctx.db->persist(documentation); + // }); + _documentations.push_back(documentation); for(int i = 0; i usageFileLoc = @@ -355,9 +403,10 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; usageAstNode->astType = model::PythonAstNode::AstType::Usage; - transaction([&, this] { - ctx.db->persist(usageAstNode); - }); + // transaction([&, this] { + // ctx.db->persist(usageAstNode); + // }); + _astNodes.push_back(usageAstNode); } for(int i = 0; iderived = cl->id; inheritance->base = getPythonEntity(boost::python::extract(baseClasses[i])).get().id; - transaction([&, this] { - ctx.db->persist(inheritance); - }); + // transaction([&, this] { + // ctx.db->persist(inheritance); + // }); + _inheritance.push_back(inheritance); } boost::python::list methods = boost::python::extract(members.attr("methods")); @@ -389,9 +439,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; - transaction([&, this] { - ctx.db->persist(classMember); - }); + // transaction([&, this] { + // ctx.db->persist(classMember); + // }); + _members.push_back(classMember); } for(int i = 0; ikind = model::PythonClassMember::Method; classMember->staticMember = true; - transaction([&, this] { - ctx.db->persist(classMember); - }); + // transaction([&, this] { + // ctx.db->persist(classMember); + // }); + _members.push_back(classMember); } for(int i = 0; ikind = model::PythonClassMember::Attribute; classMember->staticMember = false; - transaction([&, this] { - ctx.db->persist(classMember); - }); + // transaction([&, this] { + // ctx.db->persist(classMember); + // }); + _members.push_back(classMember); } for(int i = 0; ikind = model::PythonClassMember::Attribute; classMember->staticMember = true; - transaction([&, this] { - ctx.db->persist(classMember); - }); + // transaction([&, this] { + // ctx.db->persist(classMember); + // }); + _members.push_back(classMember); } for(int i = 0; ikind = model::PythonClassMember::Class; classMember->staticMember = false; - transaction([&, this] { - ctx.db->persist(classMember); - }); + // transaction([&, this] { + // ctx.db->persist(classMember); + // }); + _members.push_back(classMember); } } catch(std::exception e){ + std::cout << "Class exception:" << std::endl; std::cout << e.what() << std::endl; } } @@ -453,7 +509,6 @@ void Persistence::persistClass(boost::python::object pyClass) void Persistence::persistImport(boost::python::object pyImport) { try { - std::cout << "4" << std::endl; model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); boost::python::list importedModules = @@ -491,10 +546,12 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->importer = file; moduleImport->imported = moduleFile; - transaction([&, this] { - ctx.db->persist(moduleAstNode); - ctx.db->persist(moduleImport); - }); + // transaction([&, this] { + // ctx.db->persist(moduleAstNode); + // ctx.db->persist(moduleImport); + // }); + _astNodes.push_back(moduleAstNode); + _imports.push_back(moduleImport); } boost::python::list importDict = importedSymbols.items(); @@ -525,16 +582,19 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->imported = moduleFile; moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1][j])).get().id; - transaction([&, this] { - ctx.db->persist(moduleImport); - }); + // transaction([&, this] { + // ctx.db->persist(moduleImport); + // }); + _imports.push_back(moduleImport); } - transaction([&, this] { - ctx.db->persist(moduleAstNode); - }); + // transaction([&, this] { + // ctx.db->persist(moduleAstNode); + // }); + _astNodes.push_back(moduleAstNode); } } catch(std::exception e){ + std::cout << "Import exception:" << std::endl; std::cout << e.what() << std::endl; } } @@ -575,6 +635,21 @@ boost::optional Persistence::createFileLocFromPythonFilePosition boost::optional Persistence::getPythonEntity(const std::string& qualifiedName) { + auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); + if (varIt != _variables.end()){ + return **varIt; + } + + auto funcIt = std::find_if(_functions.begin(), _functions.end(), [&](const auto& func){ return func->qualifiedName == qualifiedName; }); + if (funcIt != _functions.end()){ + return **funcIt; + } + + auto classIt = std::find_if(_classes.begin(), _classes.end(), [&](const auto& cl){ return cl->qualifiedName == qualifiedName; }); + if (classIt != _classes.end()){ + return **classIt; + } + using EntityQuery = odb::query; using EntityResult = odb::result; @@ -593,6 +668,7 @@ typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ boost::python::class_("Persistence", boost::python::init()) .def("f", &Persistence::f) + .def("g", &Persistence::g) .def("persist_file", &Persistence::persistFile) .def("persist_variable", &Persistence::persistVariable) .def("persist_function", &Persistence::persistFunction) @@ -602,7 +678,9 @@ BOOST_PYTHON_MODULE(persistence){ PythonParser::PythonParser(ParserContext &ctx_) : AbstractParser(ctx_) {} -PythonParser::~PythonParser() {} +PythonParser::~PythonParser() +{ +} void PythonParser::markModifiedFiles() {} @@ -614,33 +692,38 @@ bool PythonParser::parse() "/home/rmfcnb/ELTE/Code-Compass-Python-Plugin/:" "/home/rmfcnb/CodeCompass/build/plugins/python/parser/:" "/usr/lib/python3.8/", 1); - Py_Initialize(); - init_module_persistence(); - boost::python::object module = boost::python::import("my_ast.python_parser"); + try{ + Py_Initialize(); + init_module_persistence(); + + boost::python::object module = boost::python::import("my_ast.python_parser"); - if(!module.is_none()){ - boost::python::object func = module.attr("parse"); + if(!module.is_none()){ + boost::python::object func = module.attr("parse"); - if(!func.is_none() && PyCallable_Check(func.ptr())){ - std::string source_path; - for (const std::string& input : _ctx.options["input"].as>()){ - if (boost::filesystem::is_directory(input)){ - source_path = input; + if(!func.is_none() && PyCallable_Check(func.ptr())){ + std::string source_path; + for (const std::string& input : _ctx.options["input"].as>()){ + if (boost::filesystem::is_directory(input)){ + source_path = input; + } } - } - if(source_path.empty()){ - std::cout << "No source path was found" << std::endl; - } else { - PersistencePtr persistencePtr(new Persistence(_ctx)); + if(source_path.empty()){ + std::cout << "No source path was found" << std::endl; + } else { + PersistencePtr persistencePtr(new Persistence(_ctx)); - func(source_path, boost::python::ptr(persistencePtr.get())); + func(source_path, boost::python::ptr(persistencePtr.get())); + } + } else { + std::cout << "Cannot find function" << std::endl; } } else { - std::cout << "Cannot find function" << std::endl; + std::cout << "Cannot import module" << std::endl; } - } else { - std::cout << "Cannot import module" << std::endl; + }catch(boost::python::error_already_set){ + PyErr_Print(); } // Py_Finalize(); diff --git a/plugins/python/service/CMakeLists.txt b/plugins/python/service/CMakeLists.txt index 6d76cb904..4ddc22a15 100644 --- a/plugins/python/service/CMakeLists.txt +++ b/plugins/python/service/CMakeLists.txt @@ -11,7 +11,8 @@ include_directories(SYSTEM ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) add_library(pythonservice SHARED - src/pythonservice.cpp) + src/pythonservice.cpp + src/plugin.cpp) target_compile_options(pythonservice PUBLIC -Wno-unknown-pragmas) diff --git a/plugins/python/service/src/plugin.cpp b/plugins/python/service/src/plugin.cpp new file mode 100644 index 000000000..016babd12 --- /dev/null +++ b/plugins/python/service/src/plugin.cpp @@ -0,0 +1,26 @@ +#include + +#include + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +extern "C" +{ + boost::program_options::options_description getOptions() + { + boost::program_options::options_description description("Python Plugin"); + return description; + } + + void registerPlugin( + const cc::webserver::ServerContext& context_, + cc::webserver::PluginHandler* pluginHandler_) + { + cc::webserver::registerPluginSimple( + context_, + pluginHandler_, + CODECOMPASS_LANGUAGE_SERVICE_FACTORY_WITH_CFG(Python), + "PythonService"); + } +} +#pragma clang diagnostic pop From ffdf54cc901acbdee2528d3be97d2aad9391e5fb Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Tue, 1 Dec 2020 23:12:28 +0100 Subject: [PATCH 10/39] Set missing Usage id's. Fix python dependecnies and narrowing error. --- parser/CMakeLists.txt | 8 ++++++- plugins/cpp/test/CMakeLists.txt | 13 ++++++++--- plugins/cpp/test/src/cppparsertest.cpp | 26 +++++++++++----------- plugins/python/parser/src/pythonparser.cpp | 26 ++++++++++++---------- util/include/util/odbtransaction.h | 4 ++-- webserver/CMakeLists.txt | 9 ++++++++ 6 files changed, 55 insertions(+), 31 deletions(-) diff --git a/parser/CMakeLists.txt b/parser/CMakeLists.txt index ce7024d7d..5843a0448 100644 --- a/parser/CMakeLists.txt +++ b/parser/CMakeLists.txt @@ -1,7 +1,12 @@ +find_package(PythonLibs REQUIRED) + +find_package (Python) + include_directories( include ${PROJECT_SOURCE_DIR}/util/include - ${PROJECT_SOURCE_DIR}/model/include) + ${PROJECT_SOURCE_DIR}/model/include + ${PYTHON_INCLUDE_DIRS}) include_directories(SYSTEM ${ODB_INCLUDE_DIRS}) @@ -19,6 +24,7 @@ target_link_libraries(CodeCompass_parser util model ${Boost_LIBRARIES} + ${PYTHON_LIBRARIES} ${ODB_LIBRARIES} ${CMAKE_DL_LIBS} magic diff --git a/plugins/cpp/test/CMakeLists.txt b/plugins/cpp/test/CMakeLists.txt index d3431f342..f31c0486d 100644 --- a/plugins/cpp/test/CMakeLists.txt +++ b/plugins/cpp/test/CMakeLists.txt @@ -1,10 +1,15 @@ +find_package(PythonLibs REQUIRED) + +find_package (Python) + include_directories( ${PLUGIN_DIR}/model/include ${PLUGIN_DIR}/service/include ${PROJECT_BINARY_DIR}/service/language/gen-cpp ${PROJECT_BINARY_DIR}/service/project/gen-cpp ${PROJECT_SOURCE_DIR}/model/include - ${PROJECT_SOURCE_DIR}/util/include) + ${PROJECT_SOURCE_DIR}/util/include + ${PYTHON_INCLUDE_DIRS}) include_directories(SYSTEM ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) @@ -19,8 +24,8 @@ add_executable(cppparsertest src/cpptest.cpp src/cppparsertest.cpp) -target_compile_options(cppservicetest PUBLIC -Wno-unknown-pragmas) -target_compile_options(cppparsertest PUBLIC -Wno-unknown-pragmas) +target_compile_options(cppservicetest PUBLIC -Wno-unknown-pragmas -Xlinker -export-dynamic) +target_compile_options(cppparsertest PUBLIC -Wno-unknown-pragmas -Xlinker -export-dynamic) target_link_libraries(cppservicetest util @@ -28,6 +33,7 @@ target_link_libraries(cppservicetest cppmodel cppservice ${Boost_LIBRARIES} + ${PYTHON_LIBRARIES} ${GTEST_BOTH_LIBRARIES} pthread) @@ -36,6 +42,7 @@ target_link_libraries(cppparsertest model cppmodel ${Boost_LIBRARIES} + ${PYTHON_LIBRARIES} ${GTEST_BOTH_LIBRARIES} pthread) diff --git a/plugins/cpp/test/src/cppparsertest.cpp b/plugins/cpp/test/src/cppparsertest.cpp index 657761500..a068d9119 100644 --- a/plugins/cpp/test/src/cppparsertest.cpp +++ b/plugins/cpp/test/src/cppparsertest.cpp @@ -355,18 +355,18 @@ TEST_F(CppParserTest, Record) switch (n.location.range.start.line) { - case -1: - EXPECT_TRUE( - // TODO: investigate the type of this. It is possibly the parameter - // of a compiler generated copy constructor or assignment operator. - // ParameterTypeLoc and ReturnTypeLoc are both listed here, - // however, only one of them is found. Earlier when we generated - // mangled names manually, the function parameter was stored, now - // with USRs we have the return type. - n.astType == model::CppAstNode::AstType::ParameterTypeLoc || - n.astType == model::CppAstNode::AstType::ReturnTypeLoc || - n.astType == model::CppAstNode::AstType::TypeLocation); - break; + // case -1: + // EXPECT_TRUE( + // // TODO: investigate the type of this. It is possibly the parameter + // // of a compiler generated copy constructor or assignment operator. + // // ParameterTypeLoc and ReturnTypeLoc are both listed here, + // // however, only one of them is found. Earlier when we generated + // // mangled names manually, the function parameter was stored, now + // // with USRs we have the return type. + // n.astType == model::CppAstNode::AstType::ParameterTypeLoc || + // n.astType == model::CppAstNode::AstType::ReturnTypeLoc || + // n.astType == model::CppAstNode::AstType::TypeLocation); + // break; case 1: // Forward declaration. EXPECT_EQ(n.astType, model::CppAstNode::AstType::Declaration); @@ -639,7 +639,7 @@ TEST_F(CppParserTest, Variable) for (const model::CppAstNode& n : astNodes) switch (n.location.range.start.line) { - case -1: // Access by compiler generated constructors. + // case -1: // Access by compiler generated constructors. case 44: // Simple access for read. EXPECT_EQ(n.astType, model::CppAstNode::AstType::Read); break; diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 0ebfe50d5..c3deedcb9 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -219,6 +219,8 @@ void Persistence::persistVariable(boost::python::object pyVariable) usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; usageAstNode->astType = model::PythonAstNode::AstType::Usage; + usageAstNode->id = model::createIdentifier(*usageAstNode); + // transaction([&, this] { // ctx.db->persist(usageAstNode); // }); @@ -227,15 +229,12 @@ void Persistence::persistVariable(boost::python::object pyVariable) } catch (const odb::object_already_persistent& ex) { - std::cout << "Var exception already persistent:" << std::endl; - std::cout << ex.what() << std::endl; + std::cout << "Var exception already persistent: " << ex.what() << std::endl; } catch (const odb::database_exception& ex) { - std::cout << "Var exception db exception:" << std::endl; - std::cout << ex.what() << std::endl; + std::cout << "Var exception db exception: " << ex.what() << std::endl; } catch(std::exception ex){ - std::cout << "Var exception:" << std::endl; - std::cout << ex.what() << std::endl; + std::cout << "Var exception: " << ex.what() << std::endl; } } @@ -320,14 +319,15 @@ void Persistence::persistFunction(boost::python::object pyFunction) usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; usageAstNode->astType = model::PythonAstNode::AstType::Usage; + usageAstNode->id = model::createIdentifier(*usageAstNode); + // transaction([&, this] { // ctx.db->persist(usageAstNode); // }); _astNodes.push_back(usageAstNode); } } catch(std::exception e){ - std::cout << "Func exception:" << std::endl; - std::cout << e.what() << std::endl; + std::cout << "Func exception:" << e.what() << std::endl; } } @@ -403,6 +403,8 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; usageAstNode->astType = model::PythonAstNode::AstType::Usage; + usageAstNode->id = model::createIdentifier(*usageAstNode); + // transaction([&, this] { // ctx.db->persist(usageAstNode); // }); @@ -501,8 +503,7 @@ void Persistence::persistClass(boost::python::object pyClass) _members.push_back(classMember); } } catch(std::exception e){ - std::cout << "Class exception:" << std::endl; - std::cout << e.what() << std::endl; + std::cout << "Class exception:" << e.what() << std::endl; } } @@ -580,6 +581,7 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; + print(boost::python::extract(import[1][j])); moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1][j])).get().id; // transaction([&, this] { @@ -594,8 +596,7 @@ void Persistence::persistImport(boost::python::object pyImport) _astNodes.push_back(moduleAstNode); } } catch(std::exception e){ - std::cout << "Import exception:" << std::endl; - std::cout << e.what() << std::endl; + std::cout << "Import exception:" << e.what() << std::endl; } } @@ -635,6 +636,7 @@ boost::optional Persistence::createFileLocFromPythonFilePosition boost::optional Persistence::getPythonEntity(const std::string& qualifiedName) { + print("qualified name: " + qualifiedName); auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); if (varIt != _variables.end()){ return **varIt; diff --git a/util/include/util/odbtransaction.h b/util/include/util/odbtransaction.h index 7640c3d14..c28a78c52 100644 --- a/util/include/util/odbtransaction.h +++ b/util/include/util/odbtransaction.h @@ -176,7 +176,7 @@ void persistAll(Cont& cont_, std::shared_ptr db_) << item->toString(); LOG(warning) << ex.what() << std::endl - << "Further changes in this transaction will be ignored!"; + << "Further changes in this transaction will be ignored! -> " << item->toString(); } catch (const odb::database_exception& ex) { @@ -190,7 +190,7 @@ void persistAll(Cont& cont_, std::shared_ptr db_) } #endif - LOG(error) << ex.what() << std::endl; + LOG(error) << ex.what() << " -> " << item->toString() << std::endl; throw; } } diff --git a/webserver/CMakeLists.txt b/webserver/CMakeLists.txt index 534f4a341..2bb216e8d 100644 --- a/webserver/CMakeLists.txt +++ b/webserver/CMakeLists.txt @@ -1,5 +1,11 @@ add_subdirectory(authenticators) +find_package(PythonLibs REQUIRED) + +find_package (Python) + +include_directories(CodeCompass_webserver ${PYTHON_INCLUDE_DIRS}) + add_executable(CodeCompass_webserver src/webserver.cpp src/authentication.cpp @@ -27,9 +33,12 @@ target_link_libraries(CodeCompass_webserver mongoose ${Boost_LIBRARIES} ${ODB_LIBRARIES} + ${PYTHON_LIBRARIES} pthread dl) +target_link_options(CodeCompass_parser PUBLIC -Xlinker -export-dynamic) + install(TARGETS CodeCompass_webserver RUNTIME DESTINATION ${INSTALL_BIN_DIR} LIBRARY DESTINATION ${INSTALL_LIB_DIR}) From a48dfb2549ba8a785bab2fa3d2c554f327377b4a Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 13 Dec 2020 17:09:23 +0100 Subject: [PATCH 11/39] Fix python service and webgui --- parser/CMakeLists.txt | 8 +- plugins/cpp/parser/src/cppparser.cpp | 2 - plugins/cpp/test/CMakeLists.txt | 13 +- plugins/cpp/test/src/cppparsertest.cpp | 26 +- plugins/cpp/webgui/js/cppInfoTree.js | 1 + .../python/model/include/model/pythonimport.h | 7 + plugins/python/parser/CMakeLists.txt | 4 + plugins/python/parser/src/pythonparser.cpp | 381 +++++++++++++++--- .../service/include/service/pythonservice.h | 8 +- plugins/python/service/src/pythonservice.cpp | 220 ++++++---- plugins/python/webgui/js/pythonInfoTree.js | 15 +- plugins/python/webgui/js/pythonMenu.js | 6 +- .../codecompass/view/component/Text.js | 13 +- webserver/CMakeLists.txt | 9 - 14 files changed, 515 insertions(+), 198 deletions(-) diff --git a/parser/CMakeLists.txt b/parser/CMakeLists.txt index 5843a0448..ce7024d7d 100644 --- a/parser/CMakeLists.txt +++ b/parser/CMakeLists.txt @@ -1,12 +1,7 @@ -find_package(PythonLibs REQUIRED) - -find_package (Python) - include_directories( include ${PROJECT_SOURCE_DIR}/util/include - ${PROJECT_SOURCE_DIR}/model/include - ${PYTHON_INCLUDE_DIRS}) + ${PROJECT_SOURCE_DIR}/model/include) include_directories(SYSTEM ${ODB_INCLUDE_DIRS}) @@ -24,7 +19,6 @@ target_link_libraries(CodeCompass_parser util model ${Boost_LIBRARIES} - ${PYTHON_LIBRARIES} ${ODB_LIBRARIES} ${CMAKE_DL_LIBS} magic diff --git a/plugins/cpp/parser/src/cppparser.cpp b/plugins/cpp/parser/src/cppparser.cpp index 98cc68d08..c7a737234 100644 --- a/plugins/cpp/parser/src/cppparser.cpp +++ b/plugins/cpp/parser/src/cppparser.cpp @@ -680,8 +680,6 @@ bool CppParser::cleanupWorker(const std::string& path_) bool CppParser::parse() { - return true; - initBuildActions(); VisitorActionFactory::init(_ctx); diff --git a/plugins/cpp/test/CMakeLists.txt b/plugins/cpp/test/CMakeLists.txt index f31c0486d..d3431f342 100644 --- a/plugins/cpp/test/CMakeLists.txt +++ b/plugins/cpp/test/CMakeLists.txt @@ -1,15 +1,10 @@ -find_package(PythonLibs REQUIRED) - -find_package (Python) - include_directories( ${PLUGIN_DIR}/model/include ${PLUGIN_DIR}/service/include ${PROJECT_BINARY_DIR}/service/language/gen-cpp ${PROJECT_BINARY_DIR}/service/project/gen-cpp ${PROJECT_SOURCE_DIR}/model/include - ${PROJECT_SOURCE_DIR}/util/include - ${PYTHON_INCLUDE_DIRS}) + ${PROJECT_SOURCE_DIR}/util/include) include_directories(SYSTEM ${THRIFT_LIBTHRIFT_INCLUDE_DIRS}) @@ -24,8 +19,8 @@ add_executable(cppparsertest src/cpptest.cpp src/cppparsertest.cpp) -target_compile_options(cppservicetest PUBLIC -Wno-unknown-pragmas -Xlinker -export-dynamic) -target_compile_options(cppparsertest PUBLIC -Wno-unknown-pragmas -Xlinker -export-dynamic) +target_compile_options(cppservicetest PUBLIC -Wno-unknown-pragmas) +target_compile_options(cppparsertest PUBLIC -Wno-unknown-pragmas) target_link_libraries(cppservicetest util @@ -33,7 +28,6 @@ target_link_libraries(cppservicetest cppmodel cppservice ${Boost_LIBRARIES} - ${PYTHON_LIBRARIES} ${GTEST_BOTH_LIBRARIES} pthread) @@ -42,7 +36,6 @@ target_link_libraries(cppparsertest model cppmodel ${Boost_LIBRARIES} - ${PYTHON_LIBRARIES} ${GTEST_BOTH_LIBRARIES} pthread) diff --git a/plugins/cpp/test/src/cppparsertest.cpp b/plugins/cpp/test/src/cppparsertest.cpp index a068d9119..657761500 100644 --- a/plugins/cpp/test/src/cppparsertest.cpp +++ b/plugins/cpp/test/src/cppparsertest.cpp @@ -355,18 +355,18 @@ TEST_F(CppParserTest, Record) switch (n.location.range.start.line) { - // case -1: - // EXPECT_TRUE( - // // TODO: investigate the type of this. It is possibly the parameter - // // of a compiler generated copy constructor or assignment operator. - // // ParameterTypeLoc and ReturnTypeLoc are both listed here, - // // however, only one of them is found. Earlier when we generated - // // mangled names manually, the function parameter was stored, now - // // with USRs we have the return type. - // n.astType == model::CppAstNode::AstType::ParameterTypeLoc || - // n.astType == model::CppAstNode::AstType::ReturnTypeLoc || - // n.astType == model::CppAstNode::AstType::TypeLocation); - // break; + case -1: + EXPECT_TRUE( + // TODO: investigate the type of this. It is possibly the parameter + // of a compiler generated copy constructor or assignment operator. + // ParameterTypeLoc and ReturnTypeLoc are both listed here, + // however, only one of them is found. Earlier when we generated + // mangled names manually, the function parameter was stored, now + // with USRs we have the return type. + n.astType == model::CppAstNode::AstType::ParameterTypeLoc || + n.astType == model::CppAstNode::AstType::ReturnTypeLoc || + n.astType == model::CppAstNode::AstType::TypeLocation); + break; case 1: // Forward declaration. EXPECT_EQ(n.astType, model::CppAstNode::AstType::Declaration); @@ -639,7 +639,7 @@ TEST_F(CppParserTest, Variable) for (const model::CppAstNode& n : astNodes) switch (n.location.range.start.line) { - // case -1: // Access by compiler generated constructors. + case -1: // Access by compiler generated constructors. case 44: // Simple access for read. EXPECT_EQ(n.astType, model::CppAstNode::AstType::Read); break; diff --git a/plugins/cpp/webgui/js/cppInfoTree.js b/plugins/cpp/webgui/js/cppInfoTree.js index e112f40b4..b2d32a1a6 100644 --- a/plugins/cpp/webgui/js/cppInfoTree.js +++ b/plugins/cpp/webgui/js/cppInfoTree.js @@ -291,6 +291,7 @@ function (model, viewHandler, util) { } var cppInfoTree = { + id: 'cpp-info-tree', render : function (elementInfo) { var ret = []; diff --git a/plugins/python/model/include/model/pythonimport.h b/plugins/python/model/include/model/pythonimport.h index bd47b4c91..bde220e5f 100644 --- a/plugins/python/model/include/model/pythonimport.h +++ b/plugins/python/model/include/model/pythonimport.h @@ -48,6 +48,13 @@ struct PythonImport typedef std::shared_ptr PythonImportPtr; +#pragma db view object(PythonImport) +struct PythonImportCount +{ + #pragma db column("count(" + PythonImport::id + ")") + std::size_t count; +}; + } } diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 62291ce69..991d1e9d0 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -25,3 +25,7 @@ target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) target_link_options(pythonparser PUBLIC -Xlinker -export-dynamic) install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) +install( + DIRECTORY ${PLUGIN_DIR}/parser/src/scripts/ + DESTINATION ${INSTALL_PARSER_DIR}/scripts/python + FILES_MATCHING PATTERN "*.py") diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index c3deedcb9..dfbdb2b7c 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -55,21 +55,27 @@ class Persistence ~Persistence(); void f() { std::cout << "C++" << std::endl; } - void g(boost::python::object o); + void cppprint(boost::python::object o); void persistFile(boost::python::object pyFile); void persistVariable(boost::python::object pyVariable); void persistFunction(boost::python::object pyFunction); + void persistPreprocessedClass(boost::python::object pyClass); void persistClass(boost::python::object pyClass); void persistImport(boost::python::object pyImport); private: boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); - boost::optional getPythonEntity(const std::string& qualifiedName); + model::PythonEntityPtr getPythonEntity(const std::string& qualifiedName); + model::PythonClassPtr getPythonClass(const std::string& qualifiedName); + model::PythonVariablePtr getPythonVariable(const std::string& qualifiedName); std::vector _astNodes; std::vector _variables; + std::map> _variableUsages; std::vector _functions; + std::map> _functionUsages; std::vector _classes; + std::map> _classUsages; std::vector _members; std::vector _inheritance; std::vector _imports; @@ -77,12 +83,30 @@ class Persistence std::vector _types; }; +template +static std::vector getKeys(const std::map& map) { + std::vector ret; + std::for_each(map.begin(), map.end(), [&ret](const auto& p){ ret.push_back(p.first); }); + return ret; +} + Persistence::~Persistence() { + std::map m; + std::vector v = getKeys(m); ctx.srcMgr.persistFiles(); (util::OdbTransaction(ctx.db))([this]{ util::persistAll(_astNodes, ctx.db); + for(auto& ast : _variableUsages){ + util::persistAll(ast.second, ctx.db); + } + for(auto& ast : _functionUsages){ + util::persistAll(ast.second, ctx.db); + } + for(auto& ast : _classUsages){ + util::persistAll(ast.second, ctx.db); + } util::persistAll(_variables, ctx.db); util::persistAll(_functions, ctx.db); util::persistAll(_classes, ctx.db); @@ -94,13 +118,14 @@ Persistence::~Persistence() }); } -void Persistence::g(boost::python::object o) { +void Persistence::cppprint(boost::python::object o) { std::string s = boost::python::extract(o); std::cout << s << std::endl; } void Persistence::persistFile(boost::python::object pyFile) { + print("file start"); try{ model::FilePtr file = nullptr; model::BuildSource buildSource; @@ -113,8 +138,10 @@ void Persistence::persistFile(boost::python::object pyFile) } else if(path.is_none()){ std::cout << "path is None..." << std::endl; } else { - buildSource.file = ctx.srcMgr.getFile(boost::python::extract(path)); - std::cout << buildSource.file << std::endl; + file = ctx.srcMgr.getFile(boost::python::extract(path)); + file->type = "PY"; + buildSource.file = file; + std::cout << buildSource.file->path << std::endl; switch(boost::python::extract(status)){ case 0: buildSource.file->parseStatus = model::File::PSNone; @@ -153,10 +180,12 @@ void Persistence::persistFile(boost::python::object pyFile) } catch(std::exception e){ std::cout << e.what() << std::endl; } + print("file end"); } void Persistence::persistVariable(boost::python::object pyVariable) { + print("var start"); try{ boost::python::object name = pyVariable.attr("name"); boost::python::object qualifiedName = pyVariable.attr("qualified_name"); @@ -182,6 +211,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) varAstNode->astType = model::PythonAstNode::AstType::Declaration; varAstNode->id = model::createIdentifier(*varAstNode); + // transaction([&, this] { // ctx.db->persist(*varAstNode); // }); @@ -197,9 +227,15 @@ void Persistence::persistVariable(boost::python::object pyVariable) // ctx.db->persist(*variable); // }); _variables.push_back(variable); + for(int i = 0; itype = getPythonEntity(boost::python::extract(types[i])).get().id; + std::string s = boost::python::extract(types[i]); + model::PythonEntityPtr t = getPythonEntity(boost::python::extract(types[i])); + if(t == nullptr){ + continue; + } + type->type = t->id; type->symbol = variable->id; // transaction([&, this] { // ctx.db->persist(type); @@ -207,6 +243,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) _types.push_back(type); } + _variableUsages[variable->id] = {}; for(int i = 0; i usageFileLoc = createFileLocFromPythonFilePosition(usages[i].attr("file_position")); @@ -224,7 +261,8 @@ void Persistence::persistVariable(boost::python::object pyVariable) // transaction([&, this] { // ctx.db->persist(usageAstNode); // }); - _astNodes.push_back(usageAstNode); + // _astNodes.push_back(usageAstNode); + _variableUsages[variable->id].push_back(usageAstNode); } } catch (const odb::object_already_persistent& ex) @@ -236,10 +274,12 @@ void Persistence::persistVariable(boost::python::object pyVariable) } catch(std::exception ex){ std::cout << "Var exception: " << ex.what() << std::endl; } + print("var end"); } void Persistence::persistFunction(boost::python::object pyFunction) { + print("func start"); try{ boost::python::object name = pyFunction.attr("name"); boost::python::object qualifiedName = pyFunction.attr("qualified_name"); @@ -253,8 +293,12 @@ void Persistence::persistFunction(boost::python::object pyFunction) boost::python::object pyDocumentation = pyFunction.attr("documentation"); - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || - types.is_none() || usages.is_none() || pyDocumentation.is_none()){ + boost::python::list params = boost::python::extract(pyFunction.attr("parameters")); + + boost::python::list locals = boost::python::extract(pyFunction.attr("locals")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || types.is_none() || + usages.is_none() || pyDocumentation.is_none() || params.is_none() || locals.is_none()){ return; } @@ -281,6 +325,25 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->id = model::createIdentifier(*function); + print(function->qualifiedName); + + for(int i = 0; i(params[i])); + if(param == nullptr){ + continue; + } + function->parameters.push_back(param); + } + + for(int i = 0; i(locals[i])); + if(local == nullptr){ + continue; + } + function->locals.push_back(local); + } + + // transaction([&, this] { // ctx.db->persist(function); // }); @@ -298,7 +361,11 @@ void Persistence::persistFunction(boost::python::object pyFunction) for(int i = 0; itype = getPythonEntity(boost::python::extract(types[i])).get().id; + model::PythonEntityPtr t = getPythonEntity(boost::python::extract(types[i])); + if(t == nullptr){ + continue; + } + type->type = t->id; type->symbol = function->id; // transaction([&, this] { @@ -307,6 +374,7 @@ void Persistence::persistFunction(boost::python::object pyFunction) _types.push_back(type); } + _functionUsages[function->id] = {}; for(int i = 0; i usageFileLoc = createFileLocFromPythonFilePosition(usages[i].attr("file_position")); @@ -324,15 +392,18 @@ void Persistence::persistFunction(boost::python::object pyFunction) // transaction([&, this] { // ctx.db->persist(usageAstNode); // }); - _astNodes.push_back(usageAstNode); + // _astNodes.push_back(usageAstNode); + _functionUsages[function->id].push_back(usageAstNode); } } catch(std::exception e){ std::cout << "Func exception:" << e.what() << std::endl; } + print("func end"); } -void Persistence::persistClass(boost::python::object pyClass) +void Persistence::persistPreprocessedClass(boost::python::object pyClass) { + print("class pre start"); try{ boost::python::object name = pyClass.attr("name"); boost::python::object qualifiedName = pyClass.attr("qualified_name"); @@ -340,16 +411,7 @@ void Persistence::persistClass(boost::python::object pyClass) boost::optional fileLoc = createFileLocFromPythonFilePosition(pyClass.attr("file_position")); - boost::python::list usages = boost::python::extract(pyClass.attr("usages")); - - boost::python::object pyDocumentation = pyClass.attr("documentation"); - - boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); - - boost::python::object members = pyClass.attr("members"); - - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || - usages.is_none() || pyDocumentation.is_none() || baseClasses.is_none() || members.is_none()){ + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none){ return; } @@ -366,6 +428,7 @@ void Persistence::persistClass(boost::python::object pyClass) // transaction([&, this] { // ctx.db->persist(classAstNode); // }); + _astNodes.push_back(classAstNode); model::PythonClassPtr cl(new model::PythonClass); @@ -381,6 +444,39 @@ void Persistence::persistClass(boost::python::object pyClass) // }); _classes.push_back(cl); + } catch(std::exception e){ + std::cout << "Preprocessed class exception:" << e.what() << std::endl; + } + print("class pre end"); +} + +void Persistence::persistClass(boost::python::object pyClass) +{ + print("class start"); + try{ + boost::python::object qualifiedName = pyClass.attr("qualified_name"); + + boost::python::list usages = boost::python::extract(pyClass.attr("usages")); + + boost::python::object pyDocumentation = pyClass.attr("documentation"); + + boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); + + boost::python::object members = pyClass.attr("members"); + + if(qualifiedName.is_none() || usages.is_none() || pyDocumentation.is_none() || + baseClasses.is_none() || members.is_none()){ + return; + } + + util::OdbTransaction transaction{ctx.db}; + + model::PythonClassPtr cl = getPythonClass(boost::python::extract(qualifiedName)); + + if (cl == nullptr){ + std::cout << "cl is none..." << std::endl; + } + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); documentation->documentation = boost::python::extract(pyDocumentation); documentation->documented = cl->id; @@ -390,7 +486,8 @@ void Persistence::persistClass(boost::python::object pyClass) // ctx.db->persist(documentation); // }); _documentations.push_back(documentation); - +std::cout << "1" << std::endl; + _classUsages[cl->id] = {}; for(int i = 0; i usageFileLoc = createFileLocFromPythonFilePosition(usages[i].attr("file_position")); @@ -408,13 +505,16 @@ void Persistence::persistClass(boost::python::object pyClass) // transaction([&, this] { // ctx.db->persist(usageAstNode); // }); - _astNodes.push_back(usageAstNode); + // _astNodes.push_back(usageAstNode); + _classUsages[cl->id].push_back(usageAstNode); } - +std::cout << "2" << std::endl; for(int i = 0; iderived = cl->id; - inheritance->base = getPythonEntity(boost::python::extract(baseClasses[i])).get().id; + std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); + std::cout << "bcqn: " << baseClassQualifiedName << std::endl; + inheritance->base = getPythonEntity(baseClassQualifiedName)->id; // transaction([&, this] { // ctx.db->persist(inheritance); @@ -432,11 +532,16 @@ void Persistence::persistClass(boost::python::object pyClass) staticAttributes.is_none() || classes.is_none()){ return; } - +std::cout << "3" << std::endl; for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(methods[i])).get().id; + std::string qualifiedName = boost::python::extract(methods[i].attr("qualified_name")); + model::PythonEntityPtr method = getPythonEntity(qualifiedName); + if(method == nullptr){ + std::cout << "method is none" << std::endl; + } + classMember->astNodeId = method->astNodeId; + classMember->memberId = method->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; @@ -445,12 +550,38 @@ void Persistence::persistClass(boost::python::object pyClass) // ctx.db->persist(classMember); // }); _members.push_back(classMember); - } + boost::python::list _usages = boost::python::extract(methods[i].attr("usages")); + std::vector& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _funcUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _funcUsages.push_back(usageAstNode); + } + } +std::cout << "4" << std::endl; for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(staticMethods[i])).get().id; + std::string qualifiedName = boost::python::extract(staticMethods[i].attr("qualified_name")); + model::PythonEntityPtr method = getPythonEntity(qualifiedName); + if(method == nullptr){ + std::cout << "smethod is none" << std::endl; + } + classMember->astNodeId = method->astNodeId; + classMember->memberId = method->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = true; @@ -459,12 +590,41 @@ void Persistence::persistClass(boost::python::object pyClass) // ctx.db->persist(classMember); // }); _members.push_back(classMember); - } + boost::python::list _usages = boost::python::extract(staticMethods[i].attr("usages")); + std::vector& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _funcUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _funcUsages.push_back(usageAstNode); + } + } +std::cout << "5" << std::endl; for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(attributes[i])).get().id; + std::string qualifiedName = boost::python::extract(attributes[i].attr("qualified_name")); + if (qualifiedName.empty()){ + continue; // TODO: import symbol in class + } + model::PythonEntityPtr attr = getPythonEntity(qualifiedName); + if(attr == nullptr){ + std::cout << "attr is none" << std::endl; + } + classMember->astNodeId = attr->astNodeId; + classMember->memberId = attr->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = false; @@ -473,12 +633,38 @@ void Persistence::persistClass(boost::python::object pyClass) // ctx.db->persist(classMember); // }); _members.push_back(classMember); - } + boost::python::list _usages = boost::python::extract(attributes[i].attr("usages")); + std::vector& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _varUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _varUsages.push_back(usageAstNode); + } + } +std::cout << "6" << std::endl; for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(staticAttributes[i])).get().id; + std::string qualifiedName = boost::python::extract(staticAttributes[i].attr("qualified_name")); + model::PythonEntityPtr attr = getPythonEntity(qualifiedName); + if(attr == nullptr){ + std::cout << "sattr is none" << std::endl; + } + classMember->astNodeId = attr->astNodeId; + classMember->memberId = attr->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = true; @@ -487,12 +673,38 @@ void Persistence::persistClass(boost::python::object pyClass) // ctx.db->persist(classMember); // }); _members.push_back(classMember); - } + boost::python::list _usages = boost::python::extract(staticAttributes[i].attr("usages")); + std::vector& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _varUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _varUsages.push_back(usageAstNode); + } + } +std::cout << "7" << std::endl; for(int i = 0; iastNodeId = classAstNode->id; - classMember->memberId = getPythonEntity(boost::python::extract(classes[i])).get().id; + std::string qualifiedName = boost::python::extract(classes[i].attr("qualified_name")); + model::PythonEntityPtr inner = getPythonEntity(qualifiedName); + if(inner == nullptr){ + std::cout << "inner is none" << std::endl; + } + classMember->astNodeId = inner->astNodeId; + classMember->memberId = inner->id; classMember->classId = cl->id; classMember->kind = model::PythonClassMember::Class; classMember->staticMember = false; @@ -501,14 +713,37 @@ void Persistence::persistClass(boost::python::object pyClass) // ctx.db->persist(classMember); // }); _members.push_back(classMember); + + boost::python::list _usages = boost::python::extract(classes[i].attr("usages")); + std::vector& _clUsages = _classUsages[cl->id]; + for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_clUsages.begin(), _clUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _clUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _clUsages.push_back(usageAstNode); + } } } catch(std::exception e){ std::cout << "Class exception:" << e.what() << std::endl; } + print("class end"); } void Persistence::persistImport(boost::python::object pyImport) { + print("imp start"); try { model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); @@ -581,8 +816,11 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; + print("imported symbol:"); print(boost::python::extract(import[1][j])); - moduleImport->importedSymbol = getPythonEntity(boost::python::extract(import[1][j])).get().id; + auto symb = getPythonEntity(boost::python::extract(import[1][j])); + print(symb == nullptr ? "symb is null" : std::to_string(symb->id)); + moduleImport->importedSymbol = symb->id; // transaction([&, this] { // ctx.db->persist(moduleImport); @@ -598,6 +836,7 @@ void Persistence::persistImport(boost::python::object pyImport) } catch(std::exception e){ std::cout << "Import exception:" << e.what() << std::endl; } + print("imp end"); } boost::optional Persistence::createFileLocFromPythonFilePosition(boost::python::object filePosition) @@ -634,35 +873,54 @@ boost::optional Persistence::createFileLocFromPythonFilePosition return fileLoc; } -boost::optional Persistence::getPythonEntity(const std::string& qualifiedName) +model::PythonEntityPtr Persistence::getPythonEntity(const std::string& qualifiedName) { - print("qualified name: " + qualifiedName); + if(qualifiedName.empty()){ + return nullptr; + } + + // print("qualified name: " + qualifiedName); auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); if (varIt != _variables.end()){ - return **varIt; + return *varIt; } auto funcIt = std::find_if(_functions.begin(), _functions.end(), [&](const auto& func){ return func->qualifiedName == qualifiedName; }); if (funcIt != _functions.end()){ - return **funcIt; + return *funcIt; } auto classIt = std::find_if(_classes.begin(), _classes.end(), [&](const auto& cl){ return cl->qualifiedName == qualifiedName; }); if (classIt != _classes.end()){ - return **classIt; + return *classIt; + } + + return nullptr; +} + +model::PythonVariablePtr Persistence::getPythonVariable(const std::string& qualifiedName) +{ + if(qualifiedName.empty()){ + return nullptr; } - using EntityQuery = odb::query; - using EntityResult = odb::result; + // print("var qualified name: " + qualifiedName); + auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); + if (varIt != _variables.end()){ + return *varIt; + } - EntityResult entity = ctx.db->query( - EntityQuery::qualifiedName == qualifiedName); + return nullptr; +} - if (entity.empty()){ - return boost::none; +model::PythonClassPtr Persistence::getPythonClass(const std::string& qualifiedName) +{ + auto classIt = std::find_if(_classes.begin(), _classes.end(), [&](const auto& cl){ return cl->qualifiedName == qualifiedName; }); + if (classIt != _classes.end()){ + return *classIt; } - return *entity.begin(); + return nullptr; } typedef boost::shared_ptr PersistencePtr; @@ -670,10 +928,11 @@ typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ boost::python::class_("Persistence", boost::python::init()) .def("f", &Persistence::f) - .def("g", &Persistence::g) + .def("print", &Persistence::cppprint) .def("persist_file", &Persistence::persistFile) .def("persist_variable", &Persistence::persistVariable) .def("persist_function", &Persistence::persistFunction) + .def("persist_preprocessed_class", &Persistence::persistPreprocessedClass) .def("persist_class", &Persistence::persistClass) .def("persist_import", &Persistence::persistImport); } @@ -690,10 +949,12 @@ bool PythonParser::cleanupDatabase() { return true; } bool PythonParser::parse() { - setenv("PYTHONPATH", - "/home/rmfcnb/ELTE/Code-Compass-Python-Plugin/:" - "/home/rmfcnb/CodeCompass/build/plugins/python/parser/:" - "/usr/lib/python3.8/", 1); + const std::string PARSER_SCRIPTS_DIR = _ctx.compassRoot + "/lib/parserplugin/scripts/python"; + if (!boost::filesystem::exists(PARSER_SCRIPTS_DIR) || !boost::filesystem::is_directory(PARSER_SCRIPTS_DIR)){ + throw std::runtime_error(PARSER_SCRIPTS_DIR + " is not a directory!"); + } + + setenv("PYTHONPATH", PARSER_SCRIPTS_DIR.c_str(), 1); try{ Py_Initialize(); @@ -754,4 +1015,4 @@ std::shared_ptr make(ParserContext& ctx_) #pragma clang diagnostic pop } -} \ No newline at end of file +} diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index 05ca1d999..d79bbd5db 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -177,7 +177,9 @@ class PythonServiceHandler : virtual public LanguageServiceIf METHOD, /*!< Members of a class. */ - NESTED_CLASS /*!< Nested classes. */ + NESTED_CLASS, /*!< Nested classes. */ + + IMPORTED_SYMBOLS }; enum FileReferenceType @@ -233,10 +235,12 @@ class PythonServiceHandler : virtual public LanguageServiceIf model::PythonEntity queryPythonEntity(const model::PythonEntityId& id); - model::PythonEntity queryPythonEntityByAstNode(const model::PythonAstNodeId& id); + model::PythonEntity queryPythonEntityByAstNode(const std::string& qualifiedName); std::map getVisibilities(const std::vector& nodes_); + bool isGeneratedVariable(const model::PythonAstNode& node_) const; + std::shared_ptr _db; util::OdbTransaction _transaction; diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 9ff7b7564..77997d8b4 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -61,7 +62,9 @@ namespace ret.__set_entityHash(cc::model::createAstNodeInfoEntityHash(astNode_)); ret.__set_astNodeType(cc::model::astTypeToString(astNode_.astType)); ret.__set_symbolType(cc::model::symbolTypeToString(astNode_.symbolType)); - ret.__set_astNodeValue(astNode_.astValue); + std::size_t start = astNode_.qualifiedName.find_last_of(".") + 1; + std::size_t size = astNode_.qualifiedName.find_last_of(":") - start; + ret.__set_astNodeValue(astNode_.qualifiedName.substr(start, size)); VisibilityMap::const_iterator it = _visibilities.find(astNode_.id); if (it != _visibilities.end()){ @@ -69,9 +72,9 @@ namespace } ret.range.range.startpos.line = astNode_.location.range.start.line; - ret.range.range.startpos.column = astNode_.location.range.start.column; + ret.range.range.startpos.column = astNode_.location.range.start.column+1; ret.range.range.endpos.line = astNode_.location.range.end.line; - ret.range.range.endpos.column = astNode_.location.range.end.column; + ret.range.range.endpos.column = astNode_.location.range.end.column+1; if (astNode_.location.file){ ret.range.file = std::to_string(astNode_.location.file.object_id()); @@ -125,21 +128,26 @@ void PythonServiceHandler::getAstNodeInfoByPosition( AstQuery::location.file == std::stoull(fpos_.file) && // StartPos <= Pos ((AstQuery::location.range.start.line == fpos_.pos.line && - AstQuery::location.range.start.column <= fpos_.pos.column) || + AstQuery::location.range.start.column <= fpos_.pos.column-1) || AstQuery::location.range.start.line < fpos_.pos.line) && // Pos < EndPos ((AstQuery::location.range.end.line == fpos_.pos.line && - AstQuery::location.range.end.column > fpos_.pos.column) || + AstQuery::location.range.end.column > fpos_.pos.column-1) || AstQuery::location.range.end.line > fpos_.pos.line)); //--- Select innermost clickable node ---// + std::cout << "pos: " << fpos_.pos.line << ", " << fpos_.pos.column << std::endl; model::Range minRange(model::Position(0, 0), model::Position()); model::PythonAstNode min; + std::cout << "numlim: " << std::numeric_limits::max() << std::endl; + + int size = 0; for (const model::PythonAstNode& node : nodes) { - if (node.location.range < minRange) + std::cout << "node(" << ++size << "): " << node.id << "..." << (std::int64_t)node.id << std::endl; + if (!isGeneratedVariable(node) && node.location.range < minRange) { min = node; minRange = node.location.range; @@ -147,8 +155,13 @@ void PythonServiceHandler::getAstNodeInfoByPosition( } return_ = _transaction([this, &min](){ - return CreateAstNodeInfo(getVisibilities({min}))(min); + std::cout << "astnodeinfobypos: 1" << std::endl; + if(min.astType == model::PythonAstNode::AstType::Declaration){ + return CreateAstNodeInfo(getVisibilities({min}))(min); + } + return CreateAstNodeInfo(getVisibilities({}))(min); }); + std::cout << "astnodeinfobypos: 2" << std::endl; }); } @@ -176,17 +189,19 @@ void PythonServiceHandler::getDocumentation( std::string& return_, const core::AstNodeId& astNodeId_) { - model::PythonAstNode node = queryPythonAstNode(astNodeId_); - model::PythonEntity entity = queryPythonEntityByAstNode(node.id); + _transaction([&, this](){ + model::PythonAstNode node = queryPythonAstNode(astNodeId_); + model::PythonEntity entity = queryPythonEntityByAstNode(node.qualifiedName); - DocResult doc = _db->query( - DocQuery::documented == entity.id); + DocResult doc = _db->query( + DocQuery::documented == entity.id); - if (doc.empty()){ - return_ = std::string(); - } else { - return_ = doc.begin()->documentation; - } + if (doc.empty()){ + return_ = std::string(); + } else { + return_ = doc.begin()->documentation; + } + }); } void PythonServiceHandler::getProperties( @@ -201,7 +216,7 @@ void PythonServiceHandler::getProperties( case model::PythonAstNode::SymbolType::Variable: { VarResult variables = _db->query( - VarQuery::astNodeId == node.id); + VarQuery::qualifiedName == node.qualifiedName); model::PythonVariable variable = *variables.begin(); return_["Name"] = variable.name; @@ -212,7 +227,7 @@ void PythonServiceHandler::getProperties( case model::PythonAstNode::SymbolType::Function: { FuncResult functions = _db->query( - FuncQuery::astNodeId == node.id); + FuncQuery::qualifiedName == node.qualifiedName); model::PythonFunction function = *functions.begin(); return_["Name"] = function.qualifiedName.substr( @@ -226,7 +241,7 @@ void PythonServiceHandler::getProperties( case model::PythonAstNode::SymbolType::Class: { ClassResult classes = _db->query( - ClassQuery::astNodeId == node.id); + ClassQuery::qualifiedName == node.qualifiedName); model::PythonClass cl = *classes.begin(); return_["Name"] = cl.name; @@ -237,11 +252,12 @@ void PythonServiceHandler::getProperties( case model::PythonAstNode::SymbolType::Module: { + std::cout << "core: " << astNodeId_ << " ast: " << (int64_t)node.id << " q: " << node.qualifiedName << std::endl; ModImpResult modules = _db->query( ModImpQuery::astNodeId == node.id); model::PythonImport module = *modules.begin(); - - return_["From"] = module.imported->filename; + std::cout << "import ok " << std::endl; + return_["From"] = module.imported.load()->filename; //return_["To"] = module.importer->filename; //return_["Symbol"] = imported symbol @@ -327,6 +343,7 @@ void PythonServiceHandler::getReferenceTypes( break; case model::PythonAstNode::SymbolType::Module: + return_["Imported symbols"] = IMPORTED_SYMBOLS; break; } } @@ -338,15 +355,14 @@ void PythonServiceHandler::getReferences( const std::vector& /*tags_*/) { std::vector nodes; - model::PythonAstNode node; + model::PythonAstNode node = queryPythonAstNode(astNodeId_); return _transaction([&, this](){ + std::cout << "ref id: " << referenceId_ << std::endl; switch (referenceId_) { case DECLARATION: - nodes = queryPythonAstNodes( - astNodeId_, - AstQuery::astType == model::PythonAstNode::AstType::Declaration); + nodes = queryDeclarations(astNodeId_); break; case USAGE: @@ -408,8 +424,6 @@ void PythonServiceHandler::getReferences( case PARAMETER: { - node = queryPythonAstNode(astNodeId_); - FuncResult functions = _db->query( FuncQuery::astNodeId == node.id); model::PythonFunction function = *functions.begin(); @@ -424,8 +438,6 @@ void PythonServiceHandler::getReferences( case LOCAL_VAR: { - node = queryPythonAstNode(astNodeId_); - FuncResult functions = _db->query( FuncQuery::astNodeId == node.id); model::PythonFunction function = *functions.begin(); @@ -440,8 +452,6 @@ void PythonServiceHandler::getReferences( case RETURN_TYPE: { - node = queryPythonAstNode(astNodeId_); - FuncResult functions = _db->query( FuncQuery::astNodeId == node.id); model::PythonFunction function = *functions.begin(); @@ -460,10 +470,8 @@ void PythonServiceHandler::getReferences( case TYPE: { - node = queryPythonAstNode(astNodeId_); - VarResult varNodes = _db->query( - VarQuery::astNodeId == node.id); + VarQuery::qualifiedName == node.qualifiedName); const model::PythonVariable& variable = *varNodes.begin(); @@ -480,11 +488,12 @@ void PythonServiceHandler::getReferences( } case INHERIT_FROM: - node = queryPythonAstNode(astNodeId_); + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); for (const model::PythonInheritance& inh : _db->query( - InhQuery::derived == node.id)) + InhQuery::derived == cl.id)) { model::PythonEntity cl = _db->query_value( EntityQuery::id == inh.base); @@ -495,16 +504,18 @@ void PythonServiceHandler::getReferences( } break; + } case INHERIT_BY: - node = queryPythonAstNode(astNodeId_); + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); for (const model::PythonInheritance& inh : _db->query( - InhQuery::base == node.id )) + InhQuery::base == cl.id )) { model::PythonEntity cl = _db->query_value( - EntityQuery::id == inh.base); + EntityQuery::id == inh.derived); AstResult result = _db->query( AstQuery::id == cl.astNodeId && AstQuery::astType == model::PythonAstNode::AstType::Declaration); @@ -512,13 +523,14 @@ void PythonServiceHandler::getReferences( } break; + } case DATA_MEMBER: { - node = queryPythonAstNode(astNodeId_); + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); for (const model::PythonClassMember& mem : _db->query( - ClassMemQuery::astNodeId == node.id && + ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Attribute)) { for (const model::PythonVariable& var : _db->query( @@ -536,15 +548,18 @@ void PythonServiceHandler::getReferences( case METHOD: { - node = queryPythonAstNode(astNodeId_); + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); + std::cout << "nodeid: " << astNodeId_ << "class: " << (std::int64_t)cl.id << std::endl; for (const model::PythonClassMember& mem : _db->query( - ClassMemQuery::astNodeId == node.id && + ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Method)) { + std::cout << "method: " << (std::int64_t)mem.memberId << std::endl; for (const model::PythonFunction& func : _db->query( FuncQuery::id == mem.memberId)) { + std::cout << "func: " << func.qualifiedName << std::endl; nodes.push_back(queryPythonAstNode(std::to_string(func.astNodeId))); } } @@ -554,10 +569,10 @@ void PythonServiceHandler::getReferences( case NESTED_CLASS: { - node = queryPythonAstNode(astNodeId_); + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); for (const model::PythonClassMember& mem : _db->query( - ClassMemQuery::astNodeId == node.id && + ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Class)) { for (const model::PythonClass& cl : _db->query( @@ -569,6 +584,24 @@ void PythonServiceHandler::getReferences( break; } + + case IMPORTED_SYMBOLS: + { + std::cout << "impast: " << (std::int64_t)node.id << std::endl; + + for (const model::PythonImport& imp : _db->query( + ModImpQuery::astNodeId == node.id && + ModImpQuery::importedSymbol != 0)) + { + std::cout << "imp: " << (std::int64_t)imp.id << std::endl; + EntityResult symb = _db->query( + EntityQuery::id == imp.importedSymbol); + + nodes.push_back(queryPythonAstNode(std::to_string(symb.begin()->astNodeId))); + } + + break; + } } std::sort(nodes.begin(), nodes.end(), compareByValue); @@ -590,6 +623,7 @@ std::int32_t PythonServiceHandler::getReferenceCount( model::PythonAstNode node = queryPythonAstNode(astNodeId_); return _transaction([&, this]() -> std::int32_t { + std::cout << "refcountid: " << referenceId_ << " ast: " << astNodeId_ << " nodeid: " << (std::int64_t)node.id << std::endl; switch (referenceId_) { case DECLARATION: @@ -637,76 +671,79 @@ std::int32_t PythonServiceHandler::getReferenceCount( case PARAMETER: return _db->query_value( - FuncQuery::astNodeId == node.id).count; + FuncQuery::qualifiedName == node.qualifiedName).count; case LOCAL_VAR: return _db->query_value( - FuncQuery::astNodeId == node.id).count; + FuncQuery::qualifiedName == node.qualifiedName).count; case RETURN_TYPE: { - node = queryPythonAstNode(astNodeId_); - FuncResult functions = _db->query( - FuncQuery::astNodeId == node.id); + FuncQuery::qualifiedName == node.qualifiedName); const model::PythonFunction& function = *functions.begin(); std::vector types = queryTypes(function); - std::int32_t result = 0; - - for(const model::PythonClass& cl : types){ - result += _db->query_value( - ClassQuery::id == function.id).count; - } - - return result; + return types.size(); } case TYPE: { - node = queryPythonAstNode(astNodeId_); - VarResult varNodes = _db->query( - VarQuery::astNodeId == node.id); + VarQuery::qualifiedName == node.qualifiedName); + + std::cout << varNodes.begin()->toString() << std::endl; const model::PythonVariable& variable = *varNodes.begin(); std::vector types = queryTypes(variable); - std::int32_t result = 0; - - for(const model::PythonClass& cl : types){ - result += _db->query_value( - ClassQuery::id == cl.id).count; - } - - return result; + return types.size(); } case INHERIT_FROM: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); return _db->query_value( - InhQuery::derived == node.id).count; + InhQuery::derived == cl.id).count; + } case INHERIT_BY: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); return _db->query_value( - InhQuery::base == node.id).count; + InhQuery::base == cl.id).count; + } case DATA_MEMBER: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); return _db->query_value( - ClassMemQuery::astNodeId == node.id && + ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Attribute).count; + } case METHOD: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); return _db->query_value( - ClassMemQuery::astNodeId == node.id && + ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Method).count; + } case NESTED_CLASS: + { + model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); return _db->query_value( - ClassMemQuery::astNodeId == node.id && + ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Class).count; + } + + case IMPORTED_SYMBOLS: + return _db->query_value( + ModImpQuery::importer == node.location.file.object_id()).count; default: return 0; @@ -886,6 +923,9 @@ void PythonServiceHandler::getSyntaxHighlight( syntax.className = symbolClass + " " + symbolClass + "-" + model::astTypeToString(node.astType); + std::cout << "syntax highlight: " << syntax.range.startpos.line << ", " << syntax.range.startpos.column + << " - " << syntax.range.endpos.line << ", " << syntax.range.endpos.column << std::endl; + return_.push_back(std::move(syntax)); } } @@ -915,6 +955,7 @@ model::PythonAstNode PythonServiceHandler::queryPythonAstNode(const core::AstNod if (!_db->find(std::stoull(astNodeId_), node)) { core::InvalidId ex; + std::cout << "Invalid PythonAstNode ID: " << astNodeId_ << std::endl; ex.__set_msg("Invalid PythonAstNode ID"); ex.__set_nodeid(astNodeId_); throw ex; @@ -931,7 +972,7 @@ std::vector PythonServiceHandler::queryPythonAstNodes( model::PythonAstNode node = queryPythonAstNode(astNodeId_); AstResult result = _db->query( - AstQuery::id == node.id && + AstQuery::qualifiedName == node.qualifiedName && AstQuery::location.range.end.line != model::Position::npos && query_); @@ -983,12 +1024,15 @@ odb::query PythonServiceHandler::astCallsQuery(const model std::vector PythonServiceHandler::queryCalls(const core::AstNodeId& astNodeId_) { + std::cout << "qc1" << std::endl; std::vector nodes = queryDeclarations(astNodeId_); if (nodes.empty()){ return nodes; } + std::cout << "qc2: " << (std::int64_t)nodes.front().id << std::endl; + model::PythonAstNode node = nodes.front(); AstResult result = _db->query(astCallsQuery(node)); @@ -1002,7 +1046,7 @@ std::size_t PythonServiceHandler::queryPythonAstNodeCount( model::PythonAstNode node = queryPythonAstNode(astNodeId_); model::PythonAstCount q = _db->query_value( - AstQuery::id == node.id && + AstQuery::qualifiedName == node.qualifiedName && AstQuery::location.range.end.line != model::Position::npos && query_); @@ -1025,11 +1069,18 @@ std::size_t PythonServiceHandler::queryCallsCount(const core::AstNodeId& astNode std::vector PythonServiceHandler::queryTypes(const model::PythonEntity& entity) { std::vector result; + std::cout << "type: " << (std::int64_t)entity.id << std::endl; for(const model::PythonType& type : _db->query(TypeQuery::symbol == entity.id)){ + std::cout << "t: " << entity.qualifiedName << " type: " << (std::int64_t)type.type << std::endl; ClassResult cl = _db->query(ClassQuery::id == type.type); + std::cout << "ttt" << std::endl; + // for(auto& t : cl){ + // std::cout << "tc: " << t.qualifiedName << std::endl; + // } result.push_back(*cl.begin()); } + std::cout << "t.." << std::endl; return result; } @@ -1040,9 +1091,9 @@ model::PythonEntity PythonServiceHandler::queryPythonEntity(const model::PythonE return *entities.begin(); } -model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const model::PythonAstNodeId& id) +model::PythonEntity PythonServiceHandler::queryPythonEntityByAstNode(const std::string& qualifiedName) { - EntityResult entities = _db->query(EntityQuery::astNodeId == id); + EntityResult entities = _db->query(EntityQuery::qualifiedName == qualifiedName); return *entities.begin(); } @@ -1057,7 +1108,8 @@ std::map PythonServiceHandler::getVisibilit case model::PythonAstNode::SymbolType::Variable: case model::PythonAstNode::SymbolType::Function: case model::PythonAstNode::SymbolType::Class: - model::PythonEntity entity = queryPythonEntityByAstNode(node.id); + std::cout << "visibility: " << (std::int64_t)node.id << std::endl; + model::PythonEntity entity = queryPythonEntityByAstNode(node.qualifiedName); visibilities[node.id] = entity.visibility; break; } @@ -1066,6 +1118,14 @@ std::map PythonServiceHandler::getVisibilit return visibilities; } +bool PythonServiceHandler::isGeneratedVariable(const model::PythonAstNode& node_) const +{ + const std::vector generatedVariables {"__dict__", "__doc__", "__module__"}; + return std::any_of(generatedVariables.begin(), generatedVariables.end(), + [&](const std::string& name){ return node_.qualifiedName.find(name) != std::string::npos; }) && + node_.astType == model::PythonAstNode::AstType::Declaration; +} + +} } } -} \ No newline at end of file diff --git a/plugins/python/webgui/js/pythonInfoTree.js b/plugins/python/webgui/js/pythonInfoTree.js index c4b289f6f..344c17f9e 100644 --- a/plugins/python/webgui/js/pythonInfoTree.js +++ b/plugins/python/webgui/js/pythonInfoTree.js @@ -6,11 +6,10 @@ function (model, viewHandler, util){ model.addService('pythonservice', 'PythonService', LanguageServiceClient); function createLabel(astNodeInfo) { - var labelValue = astNodeInfo.astNodeValue; - - var label = astNodeInfo.range.range.startpos.line + ':' + return '' + + astNodeInfo.range.range.startpos.line + ':' + astNodeInfo.range.range.startpos.column + ': ' - + labelValue + + astNodeInfo.astNodeValue + ''; } @@ -79,8 +78,7 @@ function (model, viewHandler, util){ : elementInfo.name) + ''; - var label = createTagLabels(elementInfo.tags) // no tags in python - + '' + var label = '' + rootLabel + ': ' + rootValue + ''; @@ -206,6 +204,7 @@ function (model, viewHandler, util){ } var pythonInfoTree = { + id: 'python-info-tree', render : function (elementInfo){ var ret = []; @@ -217,7 +216,7 @@ function (model, viewHandler, util){ for (var propertyName in properties) { var propertyId = propertyName.replace(/ /g, '-'); var label = - '' + propertyName + '' + + '' + propertyName + ': ' + '' + properties[propertyName] +''; ret.push({ @@ -259,4 +258,4 @@ function (model, viewHandler, util){ type : viewHandler.moduleType.InfoTree, service : model.pythonservice }); -}); \ No newline at end of file +}); diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js index 4595d0303..7f50186db 100644 --- a/plugins/python/webgui/js/pythonMenu.js +++ b/plugins/python/webgui/js/pythonMenu.js @@ -9,8 +9,8 @@ require([ function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, viewHandler){ model.addService('pythonservice', 'PythonService', LanguageServiceClient); - var getdefintion = { - id : 'python-text-getdefintion', + var getdefinition = { + id : 'python-text-getdefinition', render : function (nodeInfo, fileInfo) { return new MenuItem({ label : 'Jump to definition', @@ -27,7 +27,7 @@ function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, viewHandler){ } }; - viewHandler.registerModule(getdefintion, { + viewHandler.registerModule(getdefinition, { type : viewHandler.moduleType.TextContextMenu, service : model.pythonservice }); diff --git a/webgui/scripts/codecompass/view/component/Text.js b/webgui/scripts/codecompass/view/component/Text.js index 8e8a71e32..4350b853a 100644 --- a/webgui/scripts/codecompass/view/component/Text.js +++ b/webgui/scripts/codecompass/view/component/Text.js @@ -449,10 +449,15 @@ function (declare, domClass, dom, style, query, topic, ContentPane, Dialog, if (!astNodeInfo) return; - var refTypes = model.cppservice.getReferenceTypes(astNodeInfo.id); - var usages = model.cppservice.getReferences( - astNodeInfo.id, - refTypes['Usage']); + var service = model.getLanguageService(fileInfo.type); + if(service){ + var refTypes = service.getReferenceTypes(astNodeInfo.id); + var usages = service.getReferences( + astNodeInfo.id, + refTypes['Usage']); + } else { + var usages = []; + } this.clearSelection(); diff --git a/webserver/CMakeLists.txt b/webserver/CMakeLists.txt index 2bb216e8d..534f4a341 100644 --- a/webserver/CMakeLists.txt +++ b/webserver/CMakeLists.txt @@ -1,11 +1,5 @@ add_subdirectory(authenticators) -find_package(PythonLibs REQUIRED) - -find_package (Python) - -include_directories(CodeCompass_webserver ${PYTHON_INCLUDE_DIRS}) - add_executable(CodeCompass_webserver src/webserver.cpp src/authentication.cpp @@ -33,12 +27,9 @@ target_link_libraries(CodeCompass_webserver mongoose ${Boost_LIBRARIES} ${ODB_LIBRARIES} - ${PYTHON_LIBRARIES} pthread dl) -target_link_options(CodeCompass_parser PUBLIC -Xlinker -export-dynamic) - install(TARGETS CodeCompass_webserver RUNTIME DESTINATION ${INSTALL_BIN_DIR} LIBRARY DESTINATION ${INSTALL_LIB_DIR}) From ff6f8eb7b97acb876bdee30cb6940546ade2cd04 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Mon, 14 Dec 2020 14:06:53 +0100 Subject: [PATCH 12/39] Remove debug console writes and unused transactions. --- plugins/python/parser/src/pythonparser.cpp | 130 +++------------------ 1 file changed, 13 insertions(+), 117 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index dfbdb2b7c..50f554121 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -54,7 +54,6 @@ class Persistence Persistence(ParserContext& ctx_) : ctx(ctx_) {} ~Persistence(); - void f() { std::cout << "C++" << std::endl; } void cppprint(boost::python::object o); void persistFile(boost::python::object pyFile); void persistVariable(boost::python::object pyVariable); @@ -125,7 +124,6 @@ void Persistence::cppprint(boost::python::object o) { void Persistence::persistFile(boost::python::object pyFile) { - print("file start"); try{ model::FilePtr file = nullptr; model::BuildSource buildSource; @@ -141,7 +139,6 @@ void Persistence::persistFile(boost::python::object pyFile) file = ctx.srcMgr.getFile(boost::python::extract(path)); file->type = "PY"; buildSource.file = file; - std::cout << buildSource.file->path << std::endl; switch(boost::python::extract(status)){ case 0: buildSource.file->parseStatus = model::File::PSNone; @@ -180,12 +177,10 @@ void Persistence::persistFile(boost::python::object pyFile) } catch(std::exception e){ std::cout << e.what() << std::endl; } - print("file end"); } void Persistence::persistVariable(boost::python::object pyVariable) { - print("var start"); try{ boost::python::object name = pyVariable.attr("name"); boost::python::object qualifiedName = pyVariable.attr("qualified_name"); @@ -202,8 +197,6 @@ void Persistence::persistVariable(boost::python::object pyVariable) return; } - util::OdbTransaction transaction{ctx.db}; - model::PythonAstNodePtr varAstNode(new model::PythonAstNode); varAstNode->location = fileLoc.get(); varAstNode->qualifiedName = boost::python::extract(qualifiedName); @@ -212,9 +205,6 @@ void Persistence::persistVariable(boost::python::object pyVariable) varAstNode->id = model::createIdentifier(*varAstNode); - // transaction([&, this] { - // ctx.db->persist(*varAstNode); - // }); _astNodes.push_back(varAstNode); model::PythonVariablePtr variable(new model::PythonVariable); variable->astNodeId = varAstNode->id; @@ -223,9 +213,6 @@ void Persistence::persistVariable(boost::python::object pyVariable) variable->visibility = boost::python::extract(visibility); variable->id = model::createIdentifier(*variable); - // transaction([&, this] { - // ctx.db->persist(*variable); - // }); _variables.push_back(variable); for(int i = 0; itype = t->id; type->symbol = variable->id; - // transaction([&, this] { - // ctx.db->persist(type); - // }); _types.push_back(type); } @@ -258,10 +242,6 @@ void Persistence::persistVariable(boost::python::object pyVariable) usageAstNode->id = model::createIdentifier(*usageAstNode); - // transaction([&, this] { - // ctx.db->persist(usageAstNode); - // }); - // _astNodes.push_back(usageAstNode); _variableUsages[variable->id].push_back(usageAstNode); } @@ -274,12 +254,10 @@ void Persistence::persistVariable(boost::python::object pyVariable) } catch(std::exception ex){ std::cout << "Var exception: " << ex.what() << std::endl; } - print("var end"); } void Persistence::persistFunction(boost::python::object pyFunction) { - print("func start"); try{ boost::python::object name = pyFunction.attr("name"); boost::python::object qualifiedName = pyFunction.attr("qualified_name"); @@ -302,8 +280,6 @@ void Persistence::persistFunction(boost::python::object pyFunction) return; } - util::OdbTransaction transaction{ctx.db}; - model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); funcAstNode->location = fileLoc.get(); funcAstNode->qualifiedName = boost::python::extract(qualifiedName); @@ -312,9 +288,6 @@ void Persistence::persistFunction(boost::python::object pyFunction) funcAstNode->id = model::createIdentifier(*funcAstNode); - // transaction([&, this] { - // ctx.db->persist(funcAstNode); - // }); _astNodes.push_back(funcAstNode); model::PythonFunctionPtr function(new model::PythonFunction); @@ -325,8 +298,6 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->id = model::createIdentifier(*function); - print(function->qualifiedName); - for(int i = 0; i(params[i])); if(param == nullptr){ @@ -343,10 +314,6 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->locals.push_back(local); } - - // transaction([&, this] { - // ctx.db->persist(function); - // }); _functions.push_back(function); model::PythonDocumentationPtr documentation(new model::PythonDocumentation); @@ -354,9 +321,6 @@ void Persistence::persistFunction(boost::python::object pyFunction) documentation->documented = function->id; documentation->documentationKind = model::PythonDocumentation::Function; - // transaction([&, this] { - // ctx.db->persist(documentation); - // }); _documentations.push_back(documentation); for(int i = 0; itype = t->id; type->symbol = function->id; - // transaction([&, this] { - // ctx.db->persist(type); - // }); _types.push_back(type); } @@ -389,21 +350,15 @@ void Persistence::persistFunction(boost::python::object pyFunction) usageAstNode->id = model::createIdentifier(*usageAstNode); - // transaction([&, this] { - // ctx.db->persist(usageAstNode); - // }); - // _astNodes.push_back(usageAstNode); _functionUsages[function->id].push_back(usageAstNode); } } catch(std::exception e){ std::cout << "Func exception:" << e.what() << std::endl; } - print("func end"); } void Persistence::persistPreprocessedClass(boost::python::object pyClass) { - print("class pre start"); try{ boost::python::object name = pyClass.attr("name"); boost::python::object qualifiedName = pyClass.attr("qualified_name"); @@ -415,8 +370,6 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) return; } - util::OdbTransaction transaction{ctx.db}; - model::PythonAstNodePtr classAstNode(new model::PythonAstNode); classAstNode->location = fileLoc.get(); classAstNode->qualifiedName = boost::python::extract(qualifiedName); @@ -425,10 +378,6 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) classAstNode->id = model::createIdentifier(*classAstNode); - // transaction([&, this] { - // ctx.db->persist(classAstNode); - // }); - _astNodes.push_back(classAstNode); model::PythonClassPtr cl(new model::PythonClass); @@ -439,20 +388,15 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) cl->id = model::createIdentifier(*cl); - // transaction([&, this] { - // ctx.db->persist(cl); - // }); _classes.push_back(cl); } catch(std::exception e){ std::cout << "Preprocessed class exception:" << e.what() << std::endl; } - print("class pre end"); } void Persistence::persistClass(boost::python::object pyClass) { - print("class start"); try{ boost::python::object qualifiedName = pyClass.attr("qualified_name"); @@ -469,12 +413,10 @@ void Persistence::persistClass(boost::python::object pyClass) return; } - util::OdbTransaction transaction{ctx.db}; - model::PythonClassPtr cl = getPythonClass(boost::python::extract(qualifiedName)); if (cl == nullptr){ - std::cout << "cl is none..." << std::endl; + std::cout << "cl is none" << std::endl; } model::PythonDocumentationPtr documentation(new model::PythonDocumentation); @@ -482,11 +424,8 @@ void Persistence::persistClass(boost::python::object pyClass) documentation->documented = cl->id; documentation->documentationKind = model::PythonDocumentation::Class; - // transaction([&, this] { - // ctx.db->persist(documentation); - // }); _documentations.push_back(documentation); -std::cout << "1" << std::endl; + _classUsages[cl->id] = {}; for(int i = 0; i usageFileLoc = @@ -502,23 +441,16 @@ std::cout << "1" << std::endl; usageAstNode->id = model::createIdentifier(*usageAstNode); - // transaction([&, this] { - // ctx.db->persist(usageAstNode); - // }); - // _astNodes.push_back(usageAstNode); _classUsages[cl->id].push_back(usageAstNode); } -std::cout << "2" << std::endl; + for(int i = 0; iderived = cl->id; std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); - std::cout << "bcqn: " << baseClassQualifiedName << std::endl; + inheritance->base = getPythonEntity(baseClassQualifiedName)->id; - // transaction([&, this] { - // ctx.db->persist(inheritance); - // }); _inheritance.push_back(inheritance); } @@ -532,7 +464,7 @@ std::cout << "2" << std::endl; staticAttributes.is_none() || classes.is_none()){ return; } -std::cout << "3" << std::endl; + for(int i = 0; i(methods[i].attr("qualified_name")); @@ -546,9 +478,6 @@ std::cout << "3" << std::endl; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; - // transaction([&, this] { - // ctx.db->persist(classMember); - // }); _members.push_back(classMember); boost::python::list _usages = boost::python::extract(methods[i].attr("usages")); @@ -572,13 +501,13 @@ std::cout << "3" << std::endl; _funcUsages.push_back(usageAstNode); } } -std::cout << "4" << std::endl; + for(int i = 0; i(staticMethods[i].attr("qualified_name")); model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ - std::cout << "smethod is none" << std::endl; + std::cout << "static method is none" << std::endl; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -586,9 +515,6 @@ std::cout << "4" << std::endl; classMember->kind = model::PythonClassMember::Method; classMember->staticMember = true; - // transaction([&, this] { - // ctx.db->persist(classMember); - // }); _members.push_back(classMember); boost::python::list _usages = boost::python::extract(staticMethods[i].attr("usages")); @@ -612,7 +538,7 @@ std::cout << "4" << std::endl; _funcUsages.push_back(usageAstNode); } } -std::cout << "5" << std::endl; + for(int i = 0; i(attributes[i].attr("qualified_name")); @@ -629,9 +555,6 @@ std::cout << "5" << std::endl; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = false; - // transaction([&, this] { - // ctx.db->persist(classMember); - // }); _members.push_back(classMember); boost::python::list _usages = boost::python::extract(attributes[i].attr("usages")); @@ -655,13 +578,13 @@ std::cout << "5" << std::endl; _varUsages.push_back(usageAstNode); } } -std::cout << "6" << std::endl; + for(int i = 0; i(staticAttributes[i].attr("qualified_name")); model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ - std::cout << "sattr is none" << std::endl; + std::cout << "static attr is none" << std::endl; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -669,9 +592,6 @@ std::cout << "6" << std::endl; classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = true; - // transaction([&, this] { - // ctx.db->persist(classMember); - // }); _members.push_back(classMember); boost::python::list _usages = boost::python::extract(staticAttributes[i].attr("usages")); @@ -695,13 +615,13 @@ std::cout << "6" << std::endl; _varUsages.push_back(usageAstNode); } } -std::cout << "7" << std::endl; + for(int i = 0; i(classes[i].attr("qualified_name")); model::PythonEntityPtr inner = getPythonEntity(qualifiedName); if(inner == nullptr){ - std::cout << "inner is none" << std::endl; + std::cout << "inner class is none" << std::endl; } classMember->astNodeId = inner->astNodeId; classMember->memberId = inner->id; @@ -709,9 +629,6 @@ std::cout << "7" << std::endl; classMember->kind = model::PythonClassMember::Class; classMember->staticMember = false; - // transaction([&, this] { - // ctx.db->persist(classMember); - // }); _members.push_back(classMember); boost::python::list _usages = boost::python::extract(classes[i].attr("usages")); @@ -738,12 +655,10 @@ std::cout << "7" << std::endl; } catch(std::exception e){ std::cout << "Class exception:" << e.what() << std::endl; } - print("class end"); } void Persistence::persistImport(boost::python::object pyImport) { - print("imp start"); try { model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); @@ -757,8 +672,6 @@ void Persistence::persistImport(boost::python::object pyImport) return; } - util::OdbTransaction transaction{ctx.db}; - for (int i = 0; i < boost::python::len(importedModules); ++i) { boost::python::object importData = importedModules[i]; @@ -782,10 +695,6 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->importer = file; moduleImport->imported = moduleFile; - // transaction([&, this] { - // ctx.db->persist(moduleAstNode); - // ctx.db->persist(moduleImport); - // }); _astNodes.push_back(moduleAstNode); _imports.push_back(moduleImport); } @@ -816,27 +725,17 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; - print("imported symbol:"); - print(boost::python::extract(import[1][j])); auto symb = getPythonEntity(boost::python::extract(import[1][j])); - print(symb == nullptr ? "symb is null" : std::to_string(symb->id)); moduleImport->importedSymbol = symb->id; - // transaction([&, this] { - // ctx.db->persist(moduleImport); - // }); _imports.push_back(moduleImport); } - // transaction([&, this] { - // ctx.db->persist(moduleAstNode); - // }); _astNodes.push_back(moduleAstNode); } } catch(std::exception e){ std::cout << "Import exception:" << e.what() << std::endl; } - print("imp end"); } boost::optional Persistence::createFileLocFromPythonFilePosition(boost::python::object filePosition) @@ -879,7 +778,6 @@ model::PythonEntityPtr Persistence::getPythonEntity(const std::string& qualified return nullptr; } - // print("qualified name: " + qualifiedName); auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); if (varIt != _variables.end()){ return *varIt; @@ -904,7 +802,6 @@ model::PythonVariablePtr Persistence::getPythonVariable(const std::string& quali return nullptr; } - // print("var qualified name: " + qualifiedName); auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); if (varIt != _variables.end()){ return *varIt; @@ -927,7 +824,6 @@ typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ boost::python::class_("Persistence", boost::python::init()) - .def("f", &Persistence::f) .def("print", &Persistence::cppprint) .def("persist_file", &Persistence::persistFile) .def("persist_variable", &Persistence::persistVariable) @@ -960,7 +856,7 @@ bool PythonParser::parse() Py_Initialize(); init_module_persistence(); - boost::python::object module = boost::python::import("my_ast.python_parser"); + boost::python::object module = boost::python::import("cc_python_parser.python_parser"); if(!module.is_none()){ boost::python::object func = module.attr("parse"); From beae53aeab39ba071638bc30255d77bfaf1fcb39 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Mon, 14 Dec 2020 14:08:58 +0100 Subject: [PATCH 13/39] Remove debug console writes from pythonservice. --- plugins/python/service/src/pythonservice.cpp | 34 -------------------- 1 file changed, 34 deletions(-) diff --git a/plugins/python/service/src/pythonservice.cpp b/plugins/python/service/src/pythonservice.cpp index 77997d8b4..724f063a7 100644 --- a/plugins/python/service/src/pythonservice.cpp +++ b/plugins/python/service/src/pythonservice.cpp @@ -136,17 +136,13 @@ void PythonServiceHandler::getAstNodeInfoByPosition( AstQuery::location.range.end.line > fpos_.pos.line)); //--- Select innermost clickable node ---// - std::cout << "pos: " << fpos_.pos.line << ", " << fpos_.pos.column << std::endl; model::Range minRange(model::Position(0, 0), model::Position()); model::PythonAstNode min; - std::cout << "numlim: " << std::numeric_limits::max() << std::endl; - int size = 0; for (const model::PythonAstNode& node : nodes) { - std::cout << "node(" << ++size << "): " << node.id << "..." << (std::int64_t)node.id << std::endl; if (!isGeneratedVariable(node) && node.location.range < minRange) { min = node; @@ -155,13 +151,11 @@ void PythonServiceHandler::getAstNodeInfoByPosition( } return_ = _transaction([this, &min](){ - std::cout << "astnodeinfobypos: 1" << std::endl; if(min.astType == model::PythonAstNode::AstType::Declaration){ return CreateAstNodeInfo(getVisibilities({min}))(min); } return CreateAstNodeInfo(getVisibilities({}))(min); }); - std::cout << "astnodeinfobypos: 2" << std::endl; }); } @@ -252,14 +246,10 @@ void PythonServiceHandler::getProperties( case model::PythonAstNode::SymbolType::Module: { - std::cout << "core: " << astNodeId_ << " ast: " << (int64_t)node.id << " q: " << node.qualifiedName << std::endl; ModImpResult modules = _db->query( ModImpQuery::astNodeId == node.id); model::PythonImport module = *modules.begin(); - std::cout << "import ok " << std::endl; return_["From"] = module.imported.load()->filename; - //return_["To"] = module.importer->filename; - //return_["Symbol"] = imported symbol break; } @@ -358,7 +348,6 @@ void PythonServiceHandler::getReferences( model::PythonAstNode node = queryPythonAstNode(astNodeId_); return _transaction([&, this](){ - std::cout << "ref id: " << referenceId_ << std::endl; switch (referenceId_) { case DECLARATION: @@ -549,17 +538,14 @@ void PythonServiceHandler::getReferences( case METHOD: { model::PythonEntity cl = queryPythonEntityByAstNode(node.qualifiedName); - std::cout << "nodeid: " << astNodeId_ << "class: " << (std::int64_t)cl.id << std::endl; for (const model::PythonClassMember& mem : _db->query( ClassMemQuery::classId == cl.id && ClassMemQuery::kind == model::PythonClassMember::Kind::Method)) { - std::cout << "method: " << (std::int64_t)mem.memberId << std::endl; for (const model::PythonFunction& func : _db->query( FuncQuery::id == mem.memberId)) { - std::cout << "func: " << func.qualifiedName << std::endl; nodes.push_back(queryPythonAstNode(std::to_string(func.astNodeId))); } } @@ -587,13 +573,10 @@ void PythonServiceHandler::getReferences( case IMPORTED_SYMBOLS: { - std::cout << "impast: " << (std::int64_t)node.id << std::endl; - for (const model::PythonImport& imp : _db->query( ModImpQuery::astNodeId == node.id && ModImpQuery::importedSymbol != 0)) { - std::cout << "imp: " << (std::int64_t)imp.id << std::endl; EntityResult symb = _db->query( EntityQuery::id == imp.importedSymbol); @@ -623,7 +606,6 @@ std::int32_t PythonServiceHandler::getReferenceCount( model::PythonAstNode node = queryPythonAstNode(astNodeId_); return _transaction([&, this]() -> std::int32_t { - std::cout << "refcountid: " << referenceId_ << " ast: " << astNodeId_ << " nodeid: " << (std::int64_t)node.id << std::endl; switch (referenceId_) { case DECLARATION: @@ -694,8 +676,6 @@ std::int32_t PythonServiceHandler::getReferenceCount( VarResult varNodes = _db->query( VarQuery::qualifiedName == node.qualifiedName); - std::cout << varNodes.begin()->toString() << std::endl; - const model::PythonVariable& variable = *varNodes.begin(); std::vector types = queryTypes(variable); @@ -923,9 +903,6 @@ void PythonServiceHandler::getSyntaxHighlight( syntax.className = symbolClass + " " + symbolClass + "-" + model::astTypeToString(node.astType); - std::cout << "syntax highlight: " << syntax.range.startpos.line << ", " << syntax.range.startpos.column - << " - " << syntax.range.endpos.line << ", " << syntax.range.endpos.column << std::endl; - return_.push_back(std::move(syntax)); } } @@ -1024,15 +1001,12 @@ odb::query PythonServiceHandler::astCallsQuery(const model std::vector PythonServiceHandler::queryCalls(const core::AstNodeId& astNodeId_) { - std::cout << "qc1" << std::endl; std::vector nodes = queryDeclarations(astNodeId_); if (nodes.empty()){ return nodes; } - std::cout << "qc2: " << (std::int64_t)nodes.front().id << std::endl; - model::PythonAstNode node = nodes.front(); AstResult result = _db->query(astCallsQuery(node)); @@ -1069,18 +1043,11 @@ std::size_t PythonServiceHandler::queryCallsCount(const core::AstNodeId& astNode std::vector PythonServiceHandler::queryTypes(const model::PythonEntity& entity) { std::vector result; - std::cout << "type: " << (std::int64_t)entity.id << std::endl; for(const model::PythonType& type : _db->query(TypeQuery::symbol == entity.id)){ - std::cout << "t: " << entity.qualifiedName << " type: " << (std::int64_t)type.type << std::endl; ClassResult cl = _db->query(ClassQuery::id == type.type); - std::cout << "ttt" << std::endl; - // for(auto& t : cl){ - // std::cout << "tc: " << t.qualifiedName << std::endl; - // } result.push_back(*cl.begin()); } - std::cout << "t.." << std::endl; return result; } @@ -1108,7 +1075,6 @@ std::map PythonServiceHandler::getVisibilit case model::PythonAstNode::SymbolType::Variable: case model::PythonAstNode::SymbolType::Function: case model::PythonAstNode::SymbolType::Class: - std::cout << "visibility: " << (std::int64_t)node.id << std::endl; model::PythonEntity entity = queryPythonEntityByAstNode(node.qualifiedName); visibilities[node.id] = entity.visibility; break; From 6d3921f2f5bb8a25968032b01e29bf70c4e385a7 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Mon, 14 Dec 2020 16:58:00 +0100 Subject: [PATCH 14/39] Add python scripts --- .../src/scripts/cc_python_parser/__init__.py | 7 + .../src/scripts/cc_python_parser/base_data.py | 73 ++ .../cc_python_parser/built_in_functions.py | 616 +++++++++ .../cc_python_parser/built_in_operators.py | 324 +++++ .../cc_python_parser/built_in_types.py | 255 ++++ .../scripts/cc_python_parser/class_data.py | 61 + .../class_init_declaration.py | 14 + .../cc_python_parser/class_preprocessor.py | 88 ++ .../cc_python_parser/common/__init__.py | 1 + .../cc_python_parser/common/file_position.py | 29 + .../cc_python_parser/common/hashable_list.py | 37 + .../cc_python_parser/common/metrics.py | 100 ++ .../cc_python_parser/common/parser_tree.py | 30 + .../cc_python_parser/common/position.py | 48 + .../cc_python_parser/common/unique_list.py | 23 + .../scripts/cc_python_parser/common/utils.py | 22 + .../src/scripts/cc_python_parser/file_info.py | 67 + .../scripts/cc_python_parser/function_data.py | 52 + .../function_symbol_collector.py | 102 ++ .../function_symbol_collector_factory.py | 15 + .../scripts/cc_python_parser/import_finder.py | 27 + .../cc_python_parser/import_preprocessor.py | 173 +++ .../src/scripts/cc_python_parser/logger.py | 16 + .../src/scripts/cc_python_parser/main.py | 60 + .../member_access_collector.py | 259 ++++ .../cc_python_parser/parse_exception.py | 20 + .../src/scripts/cc_python_parser/parser.py | 166 +++ .../cc_python_parser/persistence/__init__.py | 2 + .../cc_python_parser/persistence/base_dto.py | 36 + .../persistence/build_action.py | 11 + .../persistence/build_source_target.py | 14 + .../cc_python_parser/persistence/class_dto.py | 24 + .../persistence/documented_dto.py | 3 + .../persistence/file_content_dto.py | 3 + .../cc_python_parser/persistence/file_dto.py | 23 + .../persistence/file_position_dto.py | 16 + .../persistence/function_dto.py | 14 + .../persistence/import_dto.py | 37 + .../persistence/persistence.py | 77 ++ .../persistence/variable_dto.py | 5 + .../placeholder_function_declaration_cache.py | 52 + .../cc_python_parser/preprocessed_data.py | 7 + .../cc_python_parser/preprocessed_file.py | 73 ++ .../cc_python_parser/preprocessed_function.py | 22 + .../cc_python_parser/preprocessed_variable.py | 20 + .../scripts/cc_python_parser/python_parser.py | 26 + .../src/scripts/cc_python_parser/scope.py | 223 ++++ .../scripts/cc_python_parser/scope_manager.py | 499 ++++++++ .../cc_python_parser/symbol_collector.py | 1115 +++++++++++++++++ .../symbol_collector_interface.py | 27 + .../scripts/cc_python_parser/symbol_finder.py | 10 + .../src/scripts/cc_python_parser/type_data.py | 52 + .../cc_python_parser/type_deduction.py | 610 +++++++++ .../cc_python_parser/type_hint_data.py | 23 + .../scripts/cc_python_parser/variable_data.py | 101 ++ 55 files changed, 5810 insertions(+) create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/__init__.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/base_data.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/class_data.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/position.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/utils.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/file_info.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/function_data.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/import_finder.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/logger.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/main.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/parser.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/python_parser.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/scope.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/type_data.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/variable_data.py diff --git a/plugins/python/parser/src/scripts/cc_python_parser/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/__init__.py new file mode 100644 index 000000000..7d7338820 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/__init__.py @@ -0,0 +1,7 @@ +__all__ = ["base_data", "built_in_functions", "built_in_operators", "built_in_types", "class_data", + "class_init_declaration", "class_preprocessor", "file_info", "function_data", "function_symbol_collector", + "function_symbol_collector_factory", "import_finder", "import_preprocessor", "logger", "main", + "member_access_collector", "parser", "parse_exception", "placeholder_function_declaration_cache", + "preprocessed_data", "preprocessed_file", "preprocessed_function", "preprocessed_variable", "python_parser", + "scope", "scope_manager", "symbol_collector", "symbol_collector_interface", "symbol_finder", "type_data", + "type_deduction", "variable_data"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/base_data.py b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py new file mode 100644 index 000000000..26051dc14 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py @@ -0,0 +1,73 @@ +from typing import List, TypeVar, Generic, Set, Optional + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.type_data import DeclarationType + + +class Usage: + def __init__(self, name: str, position: FilePosition): + self.name = name + self.file_position = position + + def __str__(self): + return self.name + ' (' + str(self.file_position) + ')' + + def __repr__(self): + return self.__str__() + + +class Declaration(DeclarationType): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, + declaration_type: Optional[Set[DeclarationType]] = None): + DeclarationType.__init__(self, name, qualified_name) + assert declaration_type is None or not any(isinstance(x, type) for x in declaration_type) + self.file_position: FilePosition = pos + if declaration_type is None: + self.type: Set[DeclarationType] = set() + else: + self.type: Set[DeclarationType] = declaration_type + self.usages: List[Usage] = [] + + def is_same_declaration(self, other: 'Declaration') -> bool: + return self.__eq__(other) and self.file_position == other.file_position + + def is_same_usage(self, other: Usage): + return self.name == other.name and self.file_position == other.file_position + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.qualified_name) ^ hash(self.file_position) + + def get_type_repr(self) -> str: + return '[' + ','.join({x.get_type_repr() if x is not None else 'None' for x in self.type}) + ']' + + +class TypeDeclaration(Declaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition): + super().__init__(name, qualified_name, pos) + self.type.add(self) + + +T = TypeVar('T', bound=Declaration) + + +class ImportedDeclaration(Generic[T]): + def __init__(self, declaration: T, module, pos: FilePosition): + self.imported_declaration = declaration + self.module = module + self.position = pos + + +class ReferenceDeclaration: + def __init__(self, reference: Declaration): + self.reference: Declaration = reference + + +class DocumentedType: + def __init__(self, documentation: str): + self.documentation: str = documentation diff --git a/plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py b/plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py new file mode 100644 index 000000000..064c78788 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/built_in_functions.py @@ -0,0 +1,616 @@ +import inspect +import sys +from abc import ABC, abstractmethod +from typing import Optional, Set, Union, List + +from cc_python_parser import built_in_types +from cc_python_parser.built_in_types import Boolean, Complex, Dictionary, Float, FrozenSet, Integer, File, Slice, Tuple, \ + Object, MemoryView, String, Bytes, ByteArray, Type, RangeType +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.type_data import DeclarationType + + +class BuiltInFunction(FunctionDeclaration, ABC): + def __init__(self): + file_position = FilePosition.get_empty_file_position() + qualified_name = "builtins.function." + self.get_name() + super().__init__(self.get_name(), qualified_name, file_position, [], "", self.get_type()) + if self.get_override() is None: + self.override = [] + else: + self.override: List[str] = self.get_override() + + @staticmethod + @abstractmethod + def get_name() -> str: + pass + + @staticmethod + def get_override() -> Optional[List[str]]: + return None + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return None + + def get_type_repr(self) -> str: + if self.get_type() is None: + return "" + return ','.join([x.get_type_repr() for x in self.get_type()]) + + +class FunctionDecorator(BuiltInFunction, ABC): + pass + + +class AbsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'abs' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__abs__'] + + +class AllFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'all' + + +class AnyFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'any' + + +class AsciiFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'ascii' + + +# returns binary string +class BinFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bin' + + +class BoolFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bool' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class BreakpointFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'breakpoint' + + +class BytearrayFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bytearray' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {ByteArray()} + + +class BytesFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'bytes' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Bytes()} + + +class CallableFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'callable' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class ChrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'chr' + + +class ClassmethodFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'classmethod' + + +class CompileFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'compile' + + +class ComplexFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'complex' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__complex__', '__float__', '__index__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Complex()} + + +class DelattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'delattr' + + +class DictFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'dict' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Dictionary()} + + +class DirFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'dir' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__dir__'] + + +class DivmodFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'divmod' + + +# returns enumerate object +class EnumerateFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'enumerate' + + +class EvalFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'eval' + + +class ExecFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'exec' + + +class FilterFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'filter' + + +class FloatFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'float' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Float()} + + +class FormatFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'format' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__format__'] + + +class FrozensetFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'frozenset' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {FrozenSet()} + + +class GetattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'getattr' + + +class GlobalsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'globals' + + +class HasattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hasattr' + + +class HashFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hash' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__hash__'] + + +class HelpFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'help' + + +class HexFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'hex' + + +class IdFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'id' + + +class InputFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'input' + + +class IntFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'int' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__init__', '__index__', '__truncate__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Integer()} + + +class IsinstanceFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'isinstance' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +class IssubclassFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'issubclass' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Boolean()} + + +# returns Iterator +class IterFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'iter' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__iter__'] + + +class LenFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'len' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Integer()} + + +class ListFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'list' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {built_in_types.List()} + + +class LocalsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'locals' + + +class MapFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'map' + + +class MaxFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'max' + + +class MemoryViewFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'memoryview' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {MemoryView()} + + +class MinFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'min' + + +class NextFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'next' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__next__'] + + +class ObjectFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'object' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Object()} + + +class OctFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'oct' + + +class OpenFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'open' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {File()} + + +class OrdFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'ord' + + +class PowFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'pow' + + +class PrintFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'print' + + +class PropertyFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'property' + + +class RangeFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'range' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {RangeType()} + + +class ReprFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'repr' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__repr__'] + + +# returns iterator +class ReversedFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'reversed' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__reversed__'] + + +class RoundFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'round' + + +class SetFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'set' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {built_in_types.Set()} + + +class SetattrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'setattr' + + +class SliceFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'slice' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Slice()} + + +class SortedFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'sorted' + + +class StaticmethodFunction(FunctionDecorator): + @staticmethod + def get_name() -> str: + return 'staticmethod' + + +class StrFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'str' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__str__', '__repr__'] + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {String()} + + +class SumFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'sum' + + +class SuperFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'super' + + +class TupleFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'tuple' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Tuple()} + + +class TypeFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'type' + + @staticmethod + def get_type() -> Optional[Set[Union[DeclarationType]]]: + return {Type()} + + +class VarsFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'vars' + + @staticmethod + def get_override() -> Optional[List[str]]: + return ['__dict__'] + + +class ZipFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return 'zip' + + +class ImportFunction(BuiltInFunction): + @staticmethod + def get_name() -> str: + return '__import__' + + +built_in_functions = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltInFunction) and + x is not BuiltInFunction and x is not FunctionDecorator) + + +def get_built_in_function(name: str) -> Optional[BuiltInFunction]: + bif = [x for x in built_in_functions if x[1].get_name() == name] + assert len(bif) <= 1 + if len(bif) == 0: + return None + + return bif[0][1]() + + +def get_all_built_in_functions() -> List[BuiltInFunction]: + return [bif[1]() for bif in built_in_functions] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py b/plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py new file mode 100644 index 000000000..d0bbeb902 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/built_in_operators.py @@ -0,0 +1,324 @@ +import ast +import inspect +import sys +from abc import ABC, abstractmethod +from typing import Type, List, Optional + +from cc_python_parser.built_in_types import Complex, Float, Integer, Boolean + + +class BuiltInOperator(ABC): + def __init__(self): + self.ast_type = self.get_ast_type() + self.override = self.get_override() + + @staticmethod + @abstractmethod + def get_ast_type() -> Type[ast.AST]: + pass + + @staticmethod + def get_override() -> List[str]: + return [] + + # @staticmethod + # @abstractmethod + # def get_type() -> Type: + # pass + + +class UnaryOperator(BuiltInOperator, ABC): + pass + + +class BinaryOperator(BuiltInOperator, ABC): + pass + + +class CompareOperator(BuiltInOperator, ABC): + pass + + +class BooleanOperator(BuiltInOperator, ABC): + pass + + +class AddOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Add + + @staticmethod + def get_override() -> List[str]: + return ['__add__'] + + +class SubOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Sub + + @staticmethod + def get_override() -> List[str]: + return ['__sub__'] + + +class MultOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Mult + + @staticmethod + def get_override() -> List[str]: + return ['__mul__'] + + +class MatMultOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.MatMult + + @staticmethod + def get_override() -> List[str]: + return ['__matmul__'] + + +class DivOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Div + + @staticmethod + def get_override() -> List[str]: + return ['__truediv__'] + + +class ModOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Mod + + @staticmethod + def get_override() -> List[str]: + return ['__mod__'] + + +class PowOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Pow + + @staticmethod + def get_override() -> List[str]: + return ['__pow__'] + + +class LShiftOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.LShift + + @staticmethod + def get_override() -> List[str]: + return ['__lshift__'] + + +class RShiftOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.RShift + + @staticmethod + def get_override() -> List[str]: + return ['__rshift__'] + + +class BitOrOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitOr + + @staticmethod + def get_override() -> List[str]: + return ['__or__'] + + +class BitXorOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitXor + + @staticmethod + def get_override() -> List[str]: + return ['__xor__'] + + +class BitAndOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.BitAnd + + @staticmethod + def get_override() -> List[str]: + return ['__and__'] + + +class FloorDivOperator(BinaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.FloorDiv + + @staticmethod + def get_override() -> List[str]: + return ['__floordiv__'] + + +class AndOperator(BooleanOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.And + + +class OrOperator(BooleanOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Or + + +class EqOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Eq + + @staticmethod + def get_override() -> List[str]: + return ['__eq__'] + + +class NotEqOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.NotEq + + @staticmethod + def get_override() -> List[str]: + return ['__ne__'] + + +class LtOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Lt + + @staticmethod + def get_override() -> List[str]: + return ['__lt__'] + + +class LtEOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.LtE + + @staticmethod + def get_override() -> List[str]: + return ['__le__'] + + +class GtOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Gt + + @staticmethod + def get_override() -> List[str]: + return ['__gt__'] + + +class GtEOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.GtE + + @staticmethod + def get_override() -> List[str]: + return ['__ge__'] + + +class IsOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Is + + +class IsNotOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.IsNot + + +class InOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.In + + +class NotInOperator(CompareOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.NotIn + + +class InvertOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Invert + + @staticmethod + def get_override() -> List[str]: + return ['__inv__', '__invert__'] + + +class NotOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.Not + + @staticmethod + def get_override() -> List[str]: + return ['__not__'] + + +class UAddOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.UAdd + + @staticmethod + def get_override() -> List[str]: + return ['__pos__'] + + +class USubOperator(UnaryOperator): + @staticmethod + def get_ast_type() -> Type[ast.AST]: + return ast.USub + + @staticmethod + def get_override() -> List[str]: + return ['__neg__'] + + +built_in_operators = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltInOperator) and + x is not BuiltInOperator) + +number_precedence = [Complex, Float, Integer, Boolean] + + +def get_built_in_operator(t: ast.AST) -> Optional[BuiltInOperator]: + bio = [x for x in built_in_operators if type(t) is x[1].get_ast_type()] + assert len(bio) <= 1 + if len(bio) == 0: + return None + return bio[0][1]() diff --git a/plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py b/plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py new file mode 100644 index 000000000..30371bfb7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/built_in_types.py @@ -0,0 +1,255 @@ +import sys +import typing +import inspect +from abc import ABC, abstractmethod +from typing import Optional, Any + +from cc_python_parser.base_data import TypeDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.class_dto import ClassDeclarationDTO + + +class BuiltIn(TypeDeclaration, ABC): + def __init__(self): + super().__init__(self.get_name(), "builtins.type." + self.get_name(), FilePosition.get_empty_file_position()) + + def get_type_repr(self) -> str: + return self.name + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def create_dto(self) -> ClassDeclarationDTO: + return ClassDeclarationDTO(self.name, self.qualified_name, self.file_position, + set(), [], set(), ClassDeclarationDTO.ClassMembersDTO(), "") + + @staticmethod + @abstractmethod + def get_name() -> str: + pass + + +class GenericType: + def __init__(self, types: Optional[typing.Set[Any]] = None): + if types is None: + self.types = set() + else: + self.types = types + + def add_type(self, new_type): + self.types.add(new_type) + + +class GenericBuiltInType(BuiltIn, GenericType, ABC): + def __init__(self, types: Optional[typing.Set[Any]] = None): + BuiltIn.__init__(self) + GenericType.__init__(self, types) + + def get_type_repr(self) -> str: + return self.name + '<' + ','.join({x.get_type_repr() for x in self.types}) + '>' + + +class Boolean(BuiltIn): + @staticmethod + def get_name() -> str: + return 'bool' + + +class Integer(BuiltIn): + @staticmethod + def get_name() -> str: + return 'int' + + +class Float(BuiltIn): + @staticmethod + def get_name() -> str: + return 'float' + + +class Complex(BuiltIn): + @staticmethod + def get_name() -> str: + return 'complex' + + +class String(BuiltIn): + @staticmethod + def get_name() -> str: + return 'str' + + +class List(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'list' + + +class Tuple(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'tuple' + + +class Bytes(BuiltIn): + @staticmethod + def get_name() -> str: + return 'bytes' + + +class ByteArray(BuiltIn): + @staticmethod + def get_name() -> str: + return 'bytearray' + + +class MemoryView(BuiltIn): + @staticmethod + def get_name() -> str: + return 'memoryview' + + +class Set(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'set' + + +class FrozenSet(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'frozenset' + + +class Dictionary(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'dict' + + +class RangeType(BuiltIn): # generic? (implement __index__ method) + @staticmethod + def get_name() -> str: + return 'range' + + +class NoneType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'None' + + +class EllipsisType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'Ellipsis' + + +class NotImplementedType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'NotImplemented' + + +class Slice(BuiltIn): + @staticmethod + def get_name() -> str: + return 'slice' + + +class Module(BuiltIn): + @staticmethod + def get_name() -> str: + return 'module' + + +class Function(BuiltIn): + @staticmethod + def get_name() -> str: + return 'function' + + +class Method(BuiltIn): + @staticmethod + def get_name() -> str: + return 'method' + + +class Lambda(BuiltIn): + @staticmethod + def get_name() -> str: + return 'lambda' + + +class Generator(GenericBuiltInType): + def __init__(self, types: Optional[typing.Set[Any]] = None): + super().__init__(types) + + @staticmethod + def get_name() -> str: + return 'generator' + + +class Type(BuiltIn): + @staticmethod + def get_name() -> str: + return 'type' + + +class Object(BuiltIn): + @staticmethod + def get_name() -> str: + return 'object' + + +class File(BuiltIn): + @staticmethod + def get_name() -> str: + return 'file' + + +class AnyType(BuiltIn): + @staticmethod + def get_name() -> str: + return 'Any' + + +# others: contextmanager, code (~str?), internal + +built_in_types = inspect.getmembers(sys.modules[__name__], + lambda x: inspect.isclass(x) and issubclass(x, BuiltIn) and x is not BuiltIn and + x is not GenericType and x is not GenericBuiltInType) + + +def get_built_in_type(name: str) -> Optional[BuiltIn]: + bit = [x for x in built_in_types if x[1].get_name() == name] + assert len(bit) <= 1 + if len(bit) == 0: + return None + + return bit[0][1]() + + +def get_all_built_in_types() -> typing.List[BuiltIn]: + return [bit[1]() for bit in built_in_types] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_data.py b/plugins/python/parser/src/scripts/cc_python_parser/class_data.py new file mode 100644 index 000000000..2005197de --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_data.py @@ -0,0 +1,61 @@ +from typing import List, Optional + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.base_data import TypeDeclaration, ImportedDeclaration, DocumentedType +import cc_python_parser.function_data as fd +import cc_python_parser.variable_data as vd +from cc_python_parser.persistence.base_dto import UsageDTO + +from cc_python_parser.persistence.class_dto import ClassDeclarationDTO + + +# TODO: @classmethod +class ClassDeclaration(TypeDeclaration, DocumentedType): + def __init__(self, name: str, qualified_name: str, position: FilePosition, base_classes: List['ClassDeclaration'], + documentation: str): + TypeDeclaration.__init__(self, name, qualified_name, position) + DocumentedType.__init__(self, documentation) + self.base_classes: List[ClassDeclaration] = base_classes + self.methods: List[fd.FunctionDeclaration] = [] + self.static_methods: List[fd.StaticFunctionDeclaration] = [] + self.attributes: List[vd.VariableDeclaration] = [] + self.static_attributes: List[vd.StaticVariableDeclaration] = [] + self.classes: List[ClassDeclaration] = [] + + def create_dto(self) -> ClassDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + base_classes = set() + for base_class in self.base_classes: + base_classes.add(base_class.qualified_name) + members = ClassDeclarationDTO.ClassMembersDTO() + for m in self.methods: + members.methods.append(m.create_dto()) + for sm in self.static_methods: + members.static_methods.append(sm.create_dto()) + for a in self.attributes: + members.attributes.append(a.create_dto()) + for sa in self.static_attributes: + members.static_attributes.append(sa.create_dto()) + for c in self.classes: + members.classes.append(c.create_dto()) + return ClassDeclarationDTO(self.name, self.qualified_name, self.file_position, + types, usages, base_classes, members, self.documentation) + + def get_init_method(self) -> Optional[fd.FunctionDeclaration]: + for m in reversed(self.methods): + if m.name == '__init__': + return m + return None # has default init + + def get_type_repr(self) -> str: + return self.name + + +class ImportedClassDeclaration(ClassDeclaration, ImportedDeclaration[ClassDeclaration]): + def __init__(self, name: str, pos: FilePosition, class_declaration: ClassDeclaration, module): + ClassDeclaration.__init__(self, name, "", pos, class_declaration.base_classes, "") + ImportedDeclaration.__init__(self, class_declaration, module, pos) + self.type = {class_declaration} diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py b/plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py new file mode 100644 index 000000000..466451abb --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_init_declaration.py @@ -0,0 +1,14 @@ +from typing import List, Union, Set, Optional + +import cc_python_parser.base_data as data +from cc_python_parser.built_in_types import BuiltIn +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.function_data import FunctionDeclaration + + +class ClassInitDeclaration(FunctionDeclaration): + def __init__(self, class_declaration: ClassDeclaration): + super().__init__(class_declaration.name, class_declaration.qualified_name, class_declaration.file_position, + [], "") + self.class_declaration = class_declaration diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py b/plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py new file mode 100644 index 000000000..fd260e558 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_preprocessor.py @@ -0,0 +1,88 @@ +import ast +from pathlib import PurePath +from typing import List, Dict, Optional, Union + +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import has_attr, create_range_from_ast_node +from cc_python_parser.preprocessed_data import PreprocessedDeclaration +from cc_python_parser.preprocessed_function import PreprocessedFunction +from cc_python_parser.preprocessed_variable import PreprocessedVariable + + +class PreprocessedClass(PreprocessedDeclaration): + def __init__(self, name: str, file_position: FilePosition): + super().__init__(name, file_position) + self.methods: List[PreprocessedFunction] = [] + self.attributes: List[PreprocessedVariable] = [] + self.classes: List[PreprocessedClass] = [] + self.type_usages: List[Declaration] = [] + + def append_method(self, name: str, file_position: FilePosition, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]): + self.methods.append(PreprocessedFunction(name, file_position, node)) + + def append_attribute(self, name: str, file_position: FilePosition): + self.attributes.append(PreprocessedVariable(name, file_position)) + + def append_class(self, nested_class: 'PreprocessedClass'): # 'type' -> in case of type(self) + self.classes.append(nested_class) + + +class PreprocessedClassCollector(ast.NodeVisitor): + def __init__(self, path: Optional[PurePath]): + self.path = path + self.classes: List[PreprocessedClass] = [] + self.class_list: List[int] = [] + self.class_nest_class_map: Dict[PreprocessedClass, List[ast.ClassDef]] = {} + + def append_class(self, node: ast.ClassDef): + self.class_list.append(len(self.classes)) + preprocessed_class = PreprocessedClass(node.name, FilePosition(self.path, create_range_from_ast_node(node))) + self.classes.append(preprocessed_class) + self.handle_nested_class(node, preprocessed_class) + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.append_method(member, FilePosition(self.path, create_range_from_ast_node(member))) + elif isinstance(member, ast.Assign): + self.append_attribute(member, FilePosition(self.path, create_range_from_ast_node(member))) + elif isinstance(member, ast.ClassDef): + self.append_nested_class(preprocessed_class, member) + elif isinstance(member, ast.Pass): + pass + elif isinstance(member, ast.Expr) and hasattr(member, 'value') and isinstance(member.value, ast.Constant) \ + and hasattr(member.value, "value") and isinstance(member.value.value, str): + pass # documentation comment + else: + pass + # assert False, "Unknown class member: " + str(type(member)) + + def handle_nested_class(self, node: ast.ClassDef, preprocessed_class: PreprocessedClass): + for parent_class in self.class_nest_class_map: + if node in self.class_nest_class_map[parent_class]: + parent_class.append_class(preprocessed_class) + self.class_nest_class_map[parent_class].remove(node) + + def class_processed(self): + del self.class_list[-1] + + def append_method(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef], file_position: FilePosition): + self.classes[self.class_list[-1]].append_method(node.name, file_position, node) + if node.name == '__init__': + self.visit(node) + + def append_attribute(self, node: ast.Assign, file_position: FilePosition): + for attribute in node.targets: + if hasattr(attribute, 'id'): # class variable + self.classes[self.class_list[-1]].append_attribute(attribute.id, file_position) + elif hasattr(attribute, 'attr') and has_attr(attribute, 'value.id') and attribute.value.id == 'self': + self.classes[self.class_list[-1]].append_attribute(attribute.attr, file_position) + + def append_nested_class(self, node: PreprocessedClass, member: ast.ClassDef): + if node in self.class_nest_class_map: + self.class_nest_class_map[node].append(member) + else: + self.class_nest_class_map[node] = [member] + + def visit_Assign(self, node: ast.Assign): + self.append_attribute(node, FilePosition(self.path, create_range_from_ast_node(node))) + self.generic_visit(node) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py new file mode 100644 index 000000000..1df5a03e2 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py @@ -0,0 +1 @@ +__all__ = ["hashable_list", "file_position", "parser_tree", "position", "unique_list", "utils"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py b/plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py new file mode 100644 index 000000000..de1bd0519 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/file_position.py @@ -0,0 +1,29 @@ +from pathlib import PurePath +from typing import Optional + +from cc_python_parser.common.position import Range + + +class FilePosition: + def __init__(self, file: Optional[PurePath], r: Range): + self.file: Optional[PurePath] = file # Optional -> builtins + self.range: Range = r + + def __str__(self): + return "File: " + str(self.file) + " - " + str(self.range) + + def __repr__(self): + return self.__str__() + + def __hash__(self): + return hash(str(self.file)) ^ hash(self.range) + + def __eq__(self, other): + return self.file == other.file and self.range == other.range + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def get_empty_file_position(): + return FilePosition(None, Range.get_empty_range()) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py b/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py new file mode 100644 index 000000000..91f6bdd11 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py @@ -0,0 +1,37 @@ +from collections import Counter +from typing import List, TypeVar, Generic + +T = TypeVar('T') + + +class HashableList(Generic[T], List[T]): + def __hash__(self): + return hash(e for e in self) + + def __eq__(self, other): + return isinstance(other, type(self)) and Counter(self) == Counter(other) + + def __ne__(self, other): + return not self.__eq__(other) + + +class OrderedHashableList(Generic[T], List[T]): + def __hash__(self): + return hash(e for e in self) + + def __add__(self, other): + new_list = OrderedHashableList() + new_list.extend(self) + new_list.extend(other) + return new_list + + def __eq__(self, other): + if not isinstance(other, type(self)) or len(other) != len(self): + return False + for i in range(0, len(self)): + if self[i] != other[i]: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py b/plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py new file mode 100644 index 000000000..44e7be21e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/metrics.py @@ -0,0 +1,100 @@ +import datetime +from time import time + + +class PythonParserMetrics: + def __init__(self, project: str): + self.project: str = project + self.file_count: int = 0 + self.line_count: int = 0 + self.parse_time: float = 0 + self.function_count: int = 0 + self.class_count: int = 0 + self.import_count: int = 0 + self.ast_count: int = 0 + # type metrics + + def add_file_count(self, file_count: int = 1): + self.file_count += file_count + + def add_line_count(self, line_count: int = 1): + self.line_count += line_count + + def start_parsing(self): + self.parse_time = time() + + def stop_parsing(self): + self.parse_time = time() - self.parse_time + + def add_function_count(self, function_count: int = 1): + self.function_count += function_count + + def add_class_count(self, class_count: int = 1): + self.class_count += class_count + + def add_import_count(self, import_count: int = 1): + self.import_count += import_count + + def add_ast_count(self, ast_count: int = 1): + self.ast_count += ast_count + + def init(self, project: str): + self.project = project + self.file_count = 0 + self.line_count = 0 + self.parse_time = 0 + self.function_count = 0 + self.class_count = 0 + self.import_count = 0 + self.ast_count = 0 + # type metrics + + def __add__(self, other): + if not isinstance(other, PythonParserMetrics): + return self + new = self.__copy__() + new.file_count += other.file_count + new.line_count += other.line_count + new.parse_time += other.parse_time + new.function_count += other.function_count + new.class_count += other.class_count + new.import_count += other.import_count + new.ast_count += other.ast_count + return new + + def __iadd__(self, other): + return self.__add__(other) + + def __copy__(self): + copy = PythonParserMetrics(self.project) + copy.file_count = self.file_count + copy.line_count = self.line_count + copy.parse_time = self.parse_time + copy.function_count = self.function_count + copy.class_count = self.class_count + copy.import_count = self.import_count + copy.ast_count = self.ast_count + return copy + + def __deepcopy__(self, memodict={}): + return self.__copy__() + + def __str__(self): + return f""" + Metrics: + ============= + Project: {self.project} + File count: {self.file_count} + Line count: {self.line_count} + Parse time: {datetime.timedelta(seconds = self.parse_time)} + Function count: {self.function_count} + Class count: {self.class_count} + Import count: {self.import_count} + Ast count: {self.ast_count} + """ + + def __repr__(self): + return self.__str__() + + +metrics: PythonParserMetrics = PythonParserMetrics('') diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py b/plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py new file mode 100644 index 000000000..cc0d60dab --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/parser_tree.py @@ -0,0 +1,30 @@ +import ast + + +class ParserTreeNode: + def __init__(self, node, parent: 'ParserTreeNode'): + self.node = node + self.parent = parent + self.children = [] + self.process_children() + + def process_children(self): + for child in ast.iter_child_nodes(self.node): + self.children.append(ParserTreeNode(child, self)) + + +class ParserTree: + def __init__(self, node): + self.root = ParserTreeNode(node, None) + + def find_node(self, node) -> ParserTreeNode: + return self.find_node_in_parent(self.root, node) + + def find_node_in_parent(self, parent: ParserTreeNode, node) -> ParserTreeNode: + if parent.node is node: + return parent + for child in parent.children: + n = self.find_node_in_parent(child, node) + if n is not None: + return n + return None diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/position.py b/plugins/python/parser/src/scripts/cc_python_parser/common/position.py new file mode 100644 index 000000000..f3fd389d7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/position.py @@ -0,0 +1,48 @@ +class Position: + def __init__(self, line: int, col: int): + self.line = line + self.column = col + + def __str__(self): + return "line: " + str(self.line) + ", column: " + str(self.column) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.line == other.line and self.column == other.column + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.line) ^ hash(self.column) + + @staticmethod + def get_empty_position(): + return Position(0, 0) + + +class Range: + def __init__(self, start_pos: Position, end_pos: Position): + self.start_position = start_pos + self.end_position = end_pos + + def __str__(self): + return "Start position: " + str(self.start_position) + " - End position: " + str(self.end_position) + + def __repr__(self): + return self.__str__() + + def __hash__(self): + return hash(self.start_position) ^ hash(self.end_position) + + def __eq__(self, other): + return self.start_position == other.start_position and self.end_position == other.end_position + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def get_empty_range(): + return Range(Position.get_empty_position(), Position.get_empty_position()) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py b/plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py new file mode 100644 index 000000000..952f707ae --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py @@ -0,0 +1,23 @@ +from typing import TypeVar, List, Iterable + +T = TypeVar('T') + + +class UniqueList(List[T]): + def __init__(self, seq=()): + super().__init__() + self.extend(seq) + + def append(self, obj: T) -> None: + if obj in self: + self.remove(obj) + super().append(obj) + + def extend(self, iterable: Iterable[T]) -> None: + for i in iterable: + self.append(i) + + def insert(self, index: int, obj: T) -> None: + if obj in self: + self.remove(obj) + super().insert(index, obj) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py new file mode 100644 index 000000000..4b7d0004a --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py @@ -0,0 +1,22 @@ +import ast +from typing import Final, List + +from cc_python_parser.common.position import Range, Position + + +ENCODINGS: Final[List[str]] = ['utf-8', 'iso-8859-1'] + + +def has_attr(obj, attrs) -> bool: + for attr in attrs.split("."): + if hasattr(obj, attr): + obj = getattr(obj, attr) + else: + return False + return True + + +def create_range_from_ast_node(node: ast.AST) -> Range: + start_pos = Position(node.lineno, node.col_offset) + end_pos = Position(node.end_lineno, node.end_col_offset) + return Range(start_pos, end_pos) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py new file mode 100644 index 000000000..112cf6422 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py @@ -0,0 +1,67 @@ +from enum import Enum, unique, auto +from pathlib import PurePath, Path +from typing import Optional + +from cc_python_parser.common.utils import ENCODINGS +from cc_python_parser.persistence.file_content_dto import FileContentDTO +from cc_python_parser.persistence.file_dto import FileDTO +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.symbol_collector import SymbolCollector + + +@unique +class ProcessStatus(Enum): + WAITING = auto() + PREPROCESSED = auto() + PROCESSED = auto() + + +# TODO: expr_1; import; expr_2; - correct script -> not a problem +class FileInfo: + def __init__(self, file: str, path: Optional[PurePath]): # TODO: remove optional + self.file: str = file + self.path: Optional[PurePath] = path + self.symbol_collector: Optional[SymbolCollector] = None + self.preprocessed_file: PreprocessedFile = PreprocessedFile(path) + self.status: ProcessStatus = ProcessStatus.WAITING + + def preprocess_file(self, tree) -> None: + self.preprocessed_file.visit(tree) + self.status = ProcessStatus.PREPROCESSED + + def set_variable_collector(self, variable_collector: SymbolCollector) -> None: + self.symbol_collector = variable_collector + self.status = ProcessStatus.PROCESSED + + def create_dto(self) -> FileDTO: + file_dto = FileDTO() + file_dto.path = str(self.path) + file_dto.file_name = self.file + file_dto.timestamp = Path(self.path).stat().st_mtime + file_dto.content = FileContentDTO(self.get_content(self.path)) + file_dto.parent = str(self.path.parent) + file_dto.parse_status = self.get_parse_status(self.status) + file_dto.documentation = self.preprocessed_file.documentation + return file_dto + + @staticmethod + def get_content(file: PurePath) -> str: + for e in ENCODINGS: + try: + with open(str(file), "r", encoding=e) as f: + content = f.read() + except UnicodeDecodeError: + pass + else: + return content + assert False, f"Unhandled coding in {str(file)}" + + @staticmethod + def get_parse_status(status: ProcessStatus) -> int: + if status == ProcessStatus.WAITING: + return 0 + elif status == ProcessStatus.PREPROCESSED: + return 1 + elif status == ProcessStatus.PROCESSED: + return 2 + assert False diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_data.py b/plugins/python/parser/src/scripts/cc_python_parser/function_data.py new file mode 100644 index 000000000..9beaf4c51 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_data.py @@ -0,0 +1,52 @@ +from typing import List, Optional, Set, Union + +from cc_python_parser.built_in_types import AnyType, Method, Function, Type +from cc_python_parser.common.file_position import FilePosition +import cc_python_parser.base_data as data +from cc_python_parser.persistence.base_dto import UsageDTO +from cc_python_parser.persistence.function_dto import FunctionDeclarationDTO +from cc_python_parser.type_data import DeclarationType, PlaceholderType + + +class FunctionDeclaration(data.Declaration, data.DocumentedType): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, params: List[data.Declaration], + documentation: str, func_type: Optional[Set[Union[DeclarationType]]] = None): + data.Declaration.__init__(self, name, qualified_name, pos, func_type) + data.DocumentedType.__init__(self, documentation) + self.parameters: List[data.Declaration] = params + self.local_variables: List[data.Declaration] = [] + + def create_dto(self) -> FunctionDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + for t in self.type: + if isinstance(t, PlaceholderType): + types.add(AnyType().qualified_name) + # elif isinstance(t, MethodVariableDeclaration): + # types.add(Method().qualified_name) + # elif isinstance(t, FunctionVariableDeclaration): + # types.add(Function().qualified_name) + # elif isinstance(t, TypeVariableDeclaration): + # types.add(Type().qualified_name) + else: + types.add(t.qualified_name) + params = [] + for param in self.parameters: + params.append(param.qualified_name) + local_vars = [] + for local in self.local_variables: + local_vars.append(local.qualified_name) + return FunctionDeclarationDTO(self.name, self.qualified_name, self.file_position, + types, usages, self.documentation, params, local_vars) + + +class StaticFunctionDeclaration(FunctionDeclaration): + pass + + +class ImportedFunctionDeclaration(FunctionDeclaration, data.ImportedDeclaration[FunctionDeclaration]): + def __init__(self, name: str, pos: FilePosition, func_declaration: FunctionDeclaration, module): + FunctionDeclaration.__init__(self, name, "", pos, func_declaration.parameters, "", func_declaration.type) + data.ImportedDeclaration.__init__(self, func_declaration, module, pos) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py new file mode 100644 index 000000000..861e915e0 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py @@ -0,0 +1,102 @@ +import ast +from typing import Union, Any, List, Optional, Set, Tuple + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.parser_tree import ParserTree +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.scope import FunctionScope +from cc_python_parser.symbol_collector import SymbolCollector +from cc_python_parser.symbol_collector_interface import IFunctionSymbolCollector +from cc_python_parser.type_data import DeclarationType +from cc_python_parser.type_deduction import TypeDeduction +from cc_python_parser.variable_data import VariableDeclaration + + +class TemporaryScope(FunctionScope): + pass + + +class FunctionSymbolCollector(SymbolCollector, IFunctionSymbolCollector): + def __init__(self, symbol_collector: SymbolCollector, tree: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[Tuple[DeclarationType, Optional[str]]]): + SymbolCollector.__init__(self, ParserTree(tree), symbol_collector.current_file, + symbol_collector.preprocessed_file, + symbol_collector.import_finder, + symbol_collector.scope_manager.persistence, + symbol_collector.function_symbol_collector_factory) + IFunctionSymbolCollector.__init__(self) + self.scope_manager = symbol_collector.scope_manager + self.type_deduction = TypeDeduction(self, self.scope_manager, self.preprocessed_file, + self.function_symbol_collector_factory) + self.imported_declaration_scope_map = symbol_collector.imported_declaration_scope_map + + self.arguments = arguments + self.current_function_calls = [] + if isinstance(symbol_collector, FunctionSymbolCollector): + self.current_function_calls.extend(symbol_collector.current_function_calls) + if len(symbol_collector.current_function_declaration) > 0: + self.current_function_calls.append(symbol_collector.current_function_declaration[-1]) + + def collect_symbols(self): + self.scope_manager.lock_persisting() + SymbolCollector.collect_symbols(self) + self.scope_manager.unlock_persisting() + + # TODO: handle local functions + # TODO: circular import + placeholder? + def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: + r = create_range_from_ast_node(node) + self.function = FunctionDeclaration(node.name, + self.scope_manager.get_qualified_name_from_current_scope( + node.name, r.start_position.line), + FilePosition(self.current_file, r), [], "") + + for call in reversed(self.current_function_calls): + if call.name == self.function.name and call.file_position == self.function.file_position: + return + # self.scope_manager.append_function_to_current_scope(self.function) + self.current_function_declaration.append(self.function) + + def action(): + self.collect_parameters(node) + self.generic_visit(node) + + self.scope_manager.with_scope(FunctionScope(node.name, + self.scope_manager.get_qualified_scope_name_from_current_scope(), + FilePosition.get_empty_file_position()), action) + del self.current_function_declaration[-1] + + def collect_parameters(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: + args_with_default = [] + for i in range(0, len(node.args.args)): + if i < len(node.args.defaults): + default = node.args.defaults[-(i+1)] + else: + default = None + args_with_default.append((node.args.args[-(i+1)], default)) + + arg_idx = 0 + for param in reversed(args_with_default): + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.\ + get_qualified_name_from_current_scope(param[0].arg, r.start_position.line) + new_variable = VariableDeclaration(param[0].arg, qualified_name, FilePosition(self.current_file, r)) + if param[0].arg == 'self' and self.scope_manager.is_current_scope_method() and \ + node.args.args[0] is param[0]: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + else: + # has kw -> has arg -> default + keyword = [k for k in self.arguments if k[1] == param[0].arg] + if len(keyword) == 0: + if arg_idx < len(self.arguments): + new_variable.type.update(self.type_deduction.get_current_type({self.arguments[arg_idx][0]})) + arg_idx += 1 + elif param[1] is not None: + new_variable.type.update(self.type_deduction.get_current_type( + self.type_deduction.deduct_type(param[1]))) + elif len(keyword) == 1: + new_variable.type.update(self.type_deduction.get_current_type({keyword[0][0]})) + else: + assert False + self.scope_manager.append_variable_to_current_scope(new_variable) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py new file mode 100644 index 000000000..4707a4cb9 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector_factory.py @@ -0,0 +1,15 @@ +import ast +from abc import ABC, abstractmethod +from typing import Union, List, Optional, Tuple + +from cc_python_parser.symbol_collector_interface import SymbolCollectorBase, IFunctionSymbolCollector +from cc_python_parser.type_data import DeclarationType + + +class FunctionSymbolCollectorFactory(ABC): + @abstractmethod + def get_function_symbol_collector(self, symbol_collector: SymbolCollectorBase, + func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[Tuple[DeclarationType, Optional[str]]])\ + -> IFunctionSymbolCollector: + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/import_finder.py b/plugins/python/parser/src/scripts/cc_python_parser/import_finder.py new file mode 100644 index 000000000..93eb5238e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/import_finder.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from pathlib import PurePath +from typing import Optional + + +class ImportFinder(ABC): + # Optional[FileInfo] - circular import + @abstractmethod + def get_file_by_location(self, location: PurePath) -> Optional: + pass + + # Optional[GlobalScope] - circular import + def get_global_scope_by_location(self, location: PurePath) -> Optional: + file = self.get_file_by_location(location) + if file is not None and file.symbol_collector is not None: + assert file.symbol_collector.scope_manager.get_size() == 1 + return file.symbol_collector.scope_manager.get_global_scope() + elif file is not None: + return file.preprocessed_file + return None + + def get_scope_manager_by_location(self, location: PurePath) -> Optional: + file = self.get_file_by_location(location) + if file is not None and file.symbol_collector is not None: + assert file.symbol_collector.scope_manager.get_size() == 1 + return file.symbol_collector.scope_manager + return None diff --git a/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py new file mode 100644 index 000000000..79455121e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py @@ -0,0 +1,173 @@ +import ast +from pathlib import PurePath +from typing import Optional, List, Set +from importlib import util # python 3.4 or above + +from cc_python_parser.common.position import Range +from cc_python_parser.common.utils import create_range_from_ast_node + + +class FileDependency: + def __init__(self, location: PurePath, module: str, alias: str = None): + self.location = location + self.module: str = module + self.alias: Optional[str] = alias + + def get_file(self) -> str: + module_parts = self.module.split('.') + return module_parts[-1] + '.py' + + +class ModuleImport: + def __init__(self, name: str, alias: Optional[str] = None): + self.name: str = name + self.alias: Optional[str] = alias + + +class SymbolImport: + def __init__(self, name: str, alias: Optional[str] = None): + self.name: str = name + self.alias: Optional[str] = alias + + def is_all_imported(self) -> bool: + return self.name == '*' + + +class Import: + def __init__(self, path: List[str], module: ModuleImport, imported: List[SymbolImport], + location: Optional[PurePath], r: Range, is_local: bool = False): + self.path: List[str] = path + self.module: ModuleImport = module + self.imported: List[SymbolImport] = imported + self.location: Optional[PurePath] = location + self.range = r + self.is_local = is_local + + def is_module_import(self): + return len(self.imported) == 0 + + +# TODO: need locations of files, and check if modules are among them? +# TODO: handle relative import: +# '.': from the package (parent directory) of the current module (from .'package' import 'module') +# from the __init__ (from . import 'symbol') +# '..': from the grandparent directory of the current module (and so on) +class ImportTable: + def __init__(self): + self.modules: List[Import] = [] + self.import_paths: Set[PurePath] = set() + + def append_import(self, node: ast.Import, is_local: bool = False): + self.modules.extend(self.convert_ast_import_to_import(node, is_local)) + for module in node.names: + m = None + try: + if module.name != '__main__': + m = util.find_spec(module.name) + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + pass + if m is not None and m.origin is not None: + self.import_paths.add(PurePath(m.origin)) + + def append_import_from(self, node: ast.ImportFrom, is_local: bool = False): + self.modules.extend(self.convert_ast_import_from_to_import(node, is_local)) + m = None + try: + if node.level > 0 and node.module is None: + m = util.find_spec(node.names[0].name) + elif node.level == 0 or node.module is not None: + m = util.find_spec(node.module + '.' + node.names[0].name) + else: + pass # print() + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + pass + finally: + if m is None: + try: + if node.module != '__main__': + m = util.find_spec(node.module) + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + pass + if m is not None and m.origin is not None: + self.import_paths.add(PurePath(m.origin)) + + @staticmethod + def convert_ast_import_to_import(node: ast.Import, is_local: bool = False) -> List[Import]: + imports = [] + for module in node.names: + module_parts = module.name.split('.') + module_name = module_parts[-1] + module_parts.remove(module_name) + r = create_range_from_ast_node(node) + imports.append(ImportTable.create_import(module_parts, module_name, module.asname, [], r, is_local)) + return imports + + @staticmethod + def convert_ast_import_from_to_import(node: ast.ImportFrom, is_local: bool = False) -> List[Import]: + assert len(node.names) > 0 + is_module = True + try: + if node.level > 0 and node.module is None: + a = util.find_spec(node.names[0].name) + if a is None: + pass # print() + elif (node.level == 0 or node.module is not None) and \ + util.find_spec(node.module + '.' + node.names[0].name) is None: + is_module = False + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + # v3.7: before: AttributeError, after: ModuleNotFoundError + is_module = False + imports = [] + r = create_range_from_ast_node(node) + if is_module: # modules + if node.module is None: # relative + module_path = [] + else: + module_path = node.module.split('.') + for module in node.names: + imports.append( + ImportTable.create_import(module_path, module.name, module.asname, [], r, is_local)) + else: # variables, functions + imported = [] + for module in node.names: + imported.append(SymbolImport(module.name, module.asname)) + module_parts = node.module.split('.') + module_name = module_parts[-1] + module_parts.remove(module_name) + imports.append(ImportTable.create_import(module_parts, module_name, None, imported, r, is_local)) + return imports + + @staticmethod + def create_import(path: List[str], name: str, alias: Optional[str], imported: List[SymbolImport], + r: Range, is_local: bool = False) -> Import: + location = None + module = path + module.append(name) + try: + if module[-1] == '__main__': # TODO: util.find_spec finds the module, but has no spec (exception) + spec = None + else: + spec = util.find_spec('.'.join(module)) + if spec is None: + pass # print() + # assert False, "Cannot find module: " + '.'.join(module) + elif spec.has_location: + location = spec.origin + except (AttributeError, ModuleNotFoundError, ImportError, ValueError): + # v3.7: before: AttributeError, after: ModuleNotFoundError + pass + optional_location = None + if location is not None: + optional_location = PurePath(location) + return Import(path, ModuleImport(name, alias), imported, optional_location, r, is_local) + + def get_dependencies(self) -> List[FileDependency]: + dependencies = [] + for module in self.modules: + if len(module.path) == 0: + dependencies.append(FileDependency(module.location, module.module.name, module.module.alias)) + else: + dependencies.append( + FileDependency( + module.location, '.'.join(module.path) + '.' + module.module.name, module.module.alias)) + return dependencies diff --git a/plugins/python/parser/src/scripts/cc_python_parser/logger.py b/plugins/python/parser/src/scripts/cc_python_parser/logger.py new file mode 100644 index 000000000..a4621e7e2 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/logger.py @@ -0,0 +1,16 @@ +import pathlib +import logging + +directory = pathlib.Path(__file__).resolve().parent + +logger = logging.getLogger('test') +logger.setLevel(logging.DEBUG) +file_handler = logging.FileHandler(str(directory) + '/test.log', 'w') +file_handler.setLevel(logging.DEBUG) +logger.addHandler(file_handler) + +db_logger = logging.getLogger('db_test') +db_logger.setLevel(logging.DEBUG) +db_file_handler = logging.FileHandler(str(directory) + '/db_test.log', 'w') +db_file_handler.setLevel(logging.DEBUG) +db_logger.addHandler(db_file_handler) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/main.py b/plugins/python/parser/src/scripts/cc_python_parser/main.py new file mode 100644 index 000000000..43b54ecdc --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/main.py @@ -0,0 +1,60 @@ +import os +from copy import copy +from pathlib import PurePath +from typing import Final + +from cc_python_parser.common.metrics import metrics, PythonParserMetrics +from cc_python_parser.parse_exception import ParseException +from cc_python_parser.parser import Parser +from cc_python_parser.persistence.persistence import ModelPersistence + + +def main(): + def directory_exception(path: PurePath) -> bool: + directory = os.path.basename(os.path.normpath(str(path))) + return directory.startswith('.') or directory == 'venv' + + def file_exception(path: PurePath) -> bool: + return False + + code_compass: Final[str] = "/home/rmfcnb/ELTE/Code-Compass-Python-Plugin" + code_checker: Final[str] = "/home/rmfcnb/ELTE/CodeChecker" + python_std: Final[str] = "/usr/lib/python3.8" + manim: Final[str] = "/home/rmfcnb/ELTE/manim" + numpy: Final[str] = "/home/rmfcnb/ELTE/numpy" + algo: Final[str] = "/home/rmfcnb/ELTE/TheAlgorithms" + tensorflow: Final[str] = "/home/rmfcnb/ELTE/tensorflow" + + test: Final[str] = "/home/rmfcnb/ELTE/Test" + + project_roots = [code_compass, + code_checker, + python_std, + manim, + numpy, + algo, + tensorflow] + + project_roots = [test] + + metrics_result = [] + for pr in project_roots: + metrics.init(pr) + exception = ParseException(directory_exception, file_exception) + p = Parser([pr], ModelPersistence(None), exception) + print(f"Parsing: {pr}") + p.parse() + metrics_result.append(copy(metrics)) + + all_metrics = PythonParserMetrics("ALL") + for m in metrics_result: + print(m) + all_metrics += m + + print(all_metrics) + + pass + + +if __name__ == '__main__': + main() diff --git a/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py new file mode 100644 index 000000000..db629605b --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py @@ -0,0 +1,259 @@ +import ast +from typing import Optional, List, Any, Union + + +class MemberAccessCollector(ast.NodeVisitor): + class MemberData: + def __init__(self, name: Optional[str]): + self.name: str = name + + class ConstantData(MemberData): + pass + + class AttributeData(MemberData): + pass + + class MethodData(MemberData): + class Keyword: + def __init__(self, name: str, arg): + self.name: str = name + self.argument = arg + + def __init__(self, name: Optional[str], args, keywords): + super().__init__(name) + self.arguments = args + self.keywords = keywords + + class ReturnValueData(MemberData): + def __init__(self, name: Optional[str], member_data, node: ast.AST): + super().__init__(name) + self.return_of: MemberAccessCollector.MemberData = member_data + self.node = node + + class SubscriptData(MemberData): + class Index: + def __init__(self, idx): + self.idx = idx + + class Slice: + def __init__(self, lower, upper, step): + self.lower = lower + self.upper = upper + self.step = step + + def __init__(self, name: Optional[str], sub_slice: List[Union[Index, Slice]]): + super().__init__(name) + self.slice: List[Union[MemberAccessCollector.SubscriptData.Index, + MemberAccessCollector.SubscriptData.Slice]] = sub_slice + + class OperatorData(MemberData): + def __init__(self, name: Optional[str], node: Union[ast.BoolOp, ast.BinOp, ast.UnaryOp, ast.Compare]): + super().__init__(name) + + class LambdaData(MemberData): + def __init__(self, name: Optional[str], node: ast.Lambda): + super().__init__(name) + + class IfExpressionData(MemberData): + def __init__(self, name: Optional[str], node: ast.IfExp): + super().__init__(name) + + class ContainerLiteralData(MemberData): + def __init__(self, node): + super().__init__('') + self.container = node + + def __init__(self, member: Union[ast.Call, ast.Attribute]): + self.call_list: List[MemberAccessCollector.MemberData] = [] + self.arguments = [] + self.last = False + self.visit(member) + + def generic_visit(self, node: ast.AST): + if self.last: + return + # await and starred must be skipped, and process the callable + if not isinstance(node, (ast.Await, ast.Starred)): + print('Unhandled node type: ' + str(type(node))) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Call(self, node: ast.Call) -> Any: + if self.last: + return + + def process_call(value: ast.AST): + if isinstance(value, (ast.Name, ast.Attribute, ast.Lambda, ast.IfExp)): + self.call_list.append(self.MethodData(None, self.collect_arguments(node), self.collect_keywords(node))) + elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): + self.call_list.append( + self.ReturnValueData(None, self.MethodData(None, self.collect_arguments(node), + self.collect_keywords(node)), value)) + elif isinstance(value, ast.Await): + process_call(value.value) + elif isinstance(value, ast.IfExp): + pass # TODO + else: + assert False + process_call(node.func) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Subscript(self, node: ast.Subscript) -> Any: + if self.last: + return + + def process_subscript(value: ast.AST): + if isinstance(value, (ast.Name, ast.Attribute)): + self.call_list.append(self.SubscriptData(None, self.collect_slice(node))) + elif isinstance(value, (ast.Call, ast.Subscript)) or self.is_callable_after_override(value): + self.call_list.append( + self.ReturnValueData(None, self.SubscriptData(None, self.collect_slice(node)), value)) + elif isinstance(value, ast.Await): + process_subscript(value.value) + elif isinstance(value, (ast.Tuple, ast.List, ast.Dict, ast.Set)): + self.call_list.append(self.ContainerLiteralData(value)) + else: + assert False, "Not subscriptable type: " + str(type(value)) + process_subscript(node.value) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Attribute(self, node: ast.Attribute) -> Any: + if self.last: + return + if len(self.call_list) > 0 and self.call_list[-1].name is None: + self.call_list[-1].name = node.attr + else: + self.call_list.append(self.AttributeData(node.attr)) + ast.NodeVisitor.generic_visit(self, node) + + def visit_Name(self, node: ast.Name) -> Any: + if self.last: + return + if len(self.call_list) == 0 or self.call_list[-1].name is not None: + self.call_list.append(self.AttributeData(node.id)) + else: + self.call_list[-1].name = node.id + self.last = True + + def visit_Constant(self, node: ast.Constant) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = node.value + else: + value = node.value + if isinstance(value, str): + value = '"' + value + '"' + self.call_list.append(self.ConstantData(value)) + self.last = True + + def visit_Lambda(self, node: ast.Lambda) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = 'lambda' + else: + self.call_list.append(self.LambdaData('lambda', node)) + self.last = True + + def visit_IfExp(self, node: ast.IfExp) -> Any: + if self.last: + return + if self.call_list[-1].name is None: + self.call_list[-1].name = 'ifexpression' + else: + self.call_list.append(self.IfExpressionData('IfExpression', node)) + self.last = True + + def visit_BinOp(self, node: ast.BinOp) -> Any: + self.last = True + + def visit_UnaryOp(self, node: ast.UnaryOp) -> Any: + self.last = True + + def visit_BoolOp(self, node: ast.BoolOp) -> Any: + self.last = True + + def visit_Compare(self, node: ast.Compare) -> Any: + self.last = True + + def visit_List(self, node: ast.List) -> Any: + self.last = True + + def visit_Tuple(self, node: ast.Tuple) -> Any: + self.last = True + + def visit_Dict(self, node: ast.Dict) -> Any: + self.last = True + + def visit_Set(self, node: ast.Set) -> Any: + self.last = True + + # TODO: is it necessary? + @staticmethod + def collect_arguments(call: ast.Call) -> List: + arguments = [] + for arg in call.args: + arguments.append(arg) + return arguments + + @staticmethod + def collect_keywords(call: ast.Call) -> List: + keywords = [] + for kw in call.keywords: + keywords.append(MemberAccessCollector.MethodData.Keyword(kw.arg, kw.value)) + return keywords + + @staticmethod + def collect_slice(subscript: ast.Subscript) -> List[Union[SubscriptData.Index, SubscriptData.Slice]]: + # TODO: in python 3.9 this must be changed (ast.Index, ast.ExtSlice are deprecated) + sub_slice = [] + + def process_slice(node: (ast.Index, ast.Slice, ast.ExtSlice)): + if isinstance(node, ast.Index): + sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) + elif isinstance(node, ast.Slice): + sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) + elif isinstance(node, ast.ExtSlice): # 3.9 -> ast.Tuple + for dim in node.dims: + process_slice(dim) + else: + assert False, "Unknown slice: " + str(type(node)) + + process_slice(subscript.slice) + return sub_slice + + @staticmethod + def is_callable_after_override(node: ast.AST) -> bool: + return isinstance(node, ast.BoolOp) or isinstance(node, ast.BinOp) or isinstance(node, ast.UnaryOp) or \ + isinstance(node, ast.Compare) + + +class MemberAccessCollectorIterator: + def __init__(self, member_access_collector: MemberAccessCollector): + self.mac = member_access_collector + self.index = len(self.mac.call_list) + + def __iter__(self): + self.index = len(self.mac.call_list) + return self + + def __next__(self): + if self.index <= 0: + raise StopIteration + self.index -= 1 + return self.mac.call_list[self.index] + + def get_current(self): + return self.mac.call_list[self.index] + + def get_first(self): + return self.mac.call_list[-1] + + def get_last(self): + return self.mac.call_list[0] + + def is_iteration_over(self) -> bool: + return self.index <= 0 + + def is_iteration_started(self) -> bool: + return self.index < len(self.mac.call_list) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py b/plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py new file mode 100644 index 000000000..2ff8a6448 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/parse_exception.py @@ -0,0 +1,20 @@ +from pathlib import PurePath +from typing import Callable # python 3.5 or above + + +class ParseException: + def __init__(self, is_dictionary_exception: Callable[[PurePath], bool], + is_file_exception: Callable[[PurePath], bool]): + self.is_dictionary_exception = is_dictionary_exception + self.is_file_exception = is_file_exception + + def is_dictionary_exception(self, dictionary: PurePath) -> bool: + return self.is_dictionary_exception(dictionary) + + def is_file_exception(self, file: PurePath) -> bool: + return self.is_file_exception(file) + + +class DefaultParseException(ParseException): + def __init__(self): + super().__init__(lambda x: False, lambda x: False) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parser.py b/plugins/python/parser/src/scripts/cc_python_parser/parser.py new file mode 100644 index 000000000..5c586500e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/parser.py @@ -0,0 +1,166 @@ +import ast +import os +import sys +from pathlib import PurePath +from typing import List, Optional, Union, Set, Tuple + +from cc_python_parser.common.parser_tree import ParserTree +from cc_python_parser.common.utils import ENCODINGS +from cc_python_parser.file_info import FileInfo, ProcessStatus +from cc_python_parser.function_symbol_collector import FunctionSymbolCollector +from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.logger import logger, db_logger +from cc_python_parser.parse_exception import DefaultParseException, ParseException +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.scope import GlobalScope +from cc_python_parser.symbol_collector import SymbolCollector +from cc_python_parser.built_in_types import get_all_built_in_types +from cc_python_parser.built_in_functions import get_all_built_in_functions +from cc_python_parser.common.metrics import metrics + +# TODO: builtins - dir(builtins) +# TODO: change logging to DB storing +from cc_python_parser.symbol_collector_interface import IFunctionSymbolCollector +from cc_python_parser.type_data import DeclarationType + +current_file = "" # TODO: only for debug, remove it + + +class Parser(ast.NodeVisitor, ImportFinder, FunctionSymbolCollectorFactory): + def __init__(self, directories: List[str], persistence: ModelPersistence, + exceptions: Optional[ParseException] = None): + self.directories: List[str] = directories + self.other_modules: Set[PurePath] = set() + self.persistence: ModelPersistence = persistence + if exceptions is None: + exceptions = DefaultParseException() + self.exceptions: ParseException = exceptions + self.files: List[FileInfo] = [] + self.other_module_files: List[FileInfo] = [] + self.parsing_started_files = [] + self.scope_managers = [] + self.collect_files() + + def collect_files(self): + for directory in self.directories: + sys.path.append(directory) + self.process_directory(PurePath(directory)) + + def process_directory(self, directory: PurePath): + sub_directories = [] + for element in os.listdir(str(directory)): + path = directory.joinpath(element) + if os.path.isfile(str(path)): + if self.exceptions.is_file_exception(path): + continue + if element.endswith(".py"): + self.files.append(FileInfo(element, path)) + elif os.path.isdir(str(path)): + if self.exceptions.is_dictionary_exception(path): + continue + sub_directories.append(path) + else: + assert False, "Unknown element (file, directory): " + str(path) + + for subdir in sub_directories: + self.process_directory(subdir) + + def parse(self) -> None: + metrics.start_parsing() + self.persist_builtins() + for file_info in self.files: + if file_info.symbol_collector is None: + self.parse_file(file_info) + self.persist_global_scopes() + metrics.stop_parsing() + + def parse_file(self, file_info: FileInfo) -> None: + global current_file + current_file = file_info.file + if file_info.status != ProcessStatus.WAITING or current_file[-3:] != '.py': + return + logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + db_logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + if file_info.path is None: + return + + tree = None + for e in ENCODINGS: + try: + with open(str(file_info.path), "r", encoding=e) as source: + t = ast.parse(source.read()) + source.seek(0) + metrics.add_line_count(len(source.readlines())) + metrics.add_file_count() + except UnicodeDecodeError: + pass + except FileNotFoundError: + print(f"File not found: {file_info.path}") + except SyntaxError as e: + print(f"Syntax error in file {e.filename} at (line - {e.lineno}, column - {e.offset}): {e.text}") + break + else: + tree = t + break + + if tree is None: + return + file_info.preprocess_file(tree) + self.handle_modules_outside_of_project(file_info) + for dependency in file_info.preprocessed_file.import_table.get_dependencies(): + dependency_file_info = [x for x in self.files if x.path == dependency.location] + if len(dependency_file_info) == 0: + pass + elif len(dependency_file_info) == 1: + if dependency_file_info[0].status is ProcessStatus.WAITING: + self.parse_file(dependency_file_info[0]) + current_file = file_info.file + logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + db_logger. \ + debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + else: + assert False, 'Multiple file occurrence: ' + dependency.get_file() + sc = SymbolCollector(ParserTree(tree), file_info.path, file_info.preprocessed_file, + self, self.persistence, self) + # print(current_file) + sc.collect_symbols() + file_info.set_variable_collector(sc) + assert(isinstance(sc.scope_manager.get_current_scope(), GlobalScope)) + self.scope_managers.append(sc.scope_manager) + # file_info.symbol_collector.scope_manager.persist_current_scope() + + self.persistence.persist_file(file_info.create_dto()) + + def handle_modules_outside_of_project(self, file_info: FileInfo) -> None: + for m in file_info.preprocessed_file.import_table.import_paths: + # TODO: after python 3.9 use PurePath.is_relative_to instead of 'in parents' + if not any(PurePath(mm) in m.parents for mm in self.directories) and \ + m not in self.other_modules: + self.other_modules.add(m) + new_file_info = FileInfo(m.name, m) + self.other_module_files.append(new_file_info) + self.parse_file(new_file_info) + + def get_file_by_location(self, location: PurePath) -> Optional[FileInfo]: + for file in self.files: + if file.path == location: + return file + return None + + def persist_builtins(self): + for bit in get_all_built_in_types(): + self.persistence.persist_preprocessed_class(bit.create_dto()) + + for bif in get_all_built_in_functions(): + self.persistence.persist_function(bif.create_dto()) + + def persist_global_scopes(self): + for scope_manager in self.scope_managers: + scope_manager.persist_current_scope() + + def get_function_symbol_collector(self, symbol_collector: SymbolCollector, + func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: List[Tuple[DeclarationType, Optional[str]]]) \ + -> IFunctionSymbolCollector: + return FunctionSymbolCollector(symbol_collector, func_def, arguments) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py new file mode 100644 index 000000000..f15edfb7d --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py @@ -0,0 +1,2 @@ +__all__ = ["build_action", "build_source_target", "file_dto", "file_content_dto", "persistence", "class_dto", + "function_dto", "import_dto.py", "variable_dto", "base_dto"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py new file mode 100644 index 000000000..62e26bde7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/base_dto.py @@ -0,0 +1,36 @@ +from typing import List, Set + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.file_position_dto import FilePositionDTO + +PUBLIC_DECLARATION = "public" +SEMI_PRIVATE_DECLARATION = "semiprivate" +PRIVATE_DECLARATION = "private" + + +def create_file_position_dto(file_position: FilePosition) -> FilePositionDTO: + return FilePositionDTO(str(file_position.file), file_position.range) + + +class UsageDTO: + def __init__(self, file_position: FilePosition): + self.file_position: FilePositionDTO = create_file_position_dto(file_position) + + +class DeclarationDTO: + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO]): + self.name: str = name + self.qualified_name: str = qualified_name + self.file_position: FilePositionDTO = create_file_position_dto(file_position) + self.type: List[str] = list(types) # qualified names + self.usages: List[UsageDTO] = usages + self.visibility: str = self.get_visibility() + + def get_visibility(self) -> str: + if self.name.startswith('__'): + return PRIVATE_DECLARATION + elif self.name.startswith('_'): + return SEMI_PRIVATE_DECLARATION + else: + return PUBLIC_DECLARATION diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py new file mode 100644 index 000000000..6a5b428b7 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_action.py @@ -0,0 +1,11 @@ +from typing import List + +import cc_python_parser.persistence.build_source_target as bst + + +class BuildAction: + def __init__(self, command: str, sources: List[bst.BuildSource], targets: List[bst.BuildTarget]): + self.command = command + self.type = 0 # compile + self.build_sources: List[bst.BuildSource] = sources + self.build_target: List[bst.BuildTarget] = targets diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py new file mode 100644 index 000000000..21accab05 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/build_source_target.py @@ -0,0 +1,14 @@ +import cc_python_parser.persistence.build_action as ba +from cc_python_parser.persistence.file_dto import FileDTO + + +class BuildSource: + def __init__(self, file: FileDTO, action: ba.BuildAction): + self.file: FileDTO = file + self.action: ba.BuildAction = action + + +class BuildTarget: + def __init__(self, file: FileDTO, action: ba.BuildAction): + self.file: FileDTO = file + self.action: ba.BuildAction = action diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py new file mode 100644 index 000000000..4fc74f0c1 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py @@ -0,0 +1,24 @@ +from typing import Set, List + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.base_dto import UsageDTO, DeclarationDTO +from cc_python_parser.persistence.documented_dto import DocumentedDTO +from cc_python_parser.persistence.variable_dto import VariableDeclarationDTO +from cc_python_parser.persistence.function_dto import FunctionDeclarationDTO + + +class ClassDeclarationDTO(DeclarationDTO, DocumentedDTO): + class ClassMembersDTO: + def __init__(self): + self.methods: List[FunctionDTO] = [] + self.static_methods: List[FunctionDTO] = [] + self.attributes: List[VariableDTO] = [] + self.static_attributes: List[VariableDTO] = [] + self.classes: List[ClassDeclarationDTO] = [] + + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO], base_classes: Set[str], members: 'ClassDeclarationDTO.ClassMembersDTO', documentation: str): + DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) + DocumentedDTO.__init__(self, documentation) + self.base_classes: List[str] = list(base_classes) + self.members: ClassDeclarationDTO.ClassMembersDTO = members diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py new file mode 100644 index 000000000..271e09b67 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/documented_dto.py @@ -0,0 +1,3 @@ +class DocumentedDTO: + def __init__(self, documentation: str): + self.documentation: str = documentation diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py new file mode 100644 index 000000000..1487d9259 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_content_dto.py @@ -0,0 +1,3 @@ +class FileContentDTO: + def __init__(self, content: str): + self.content: str = content diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py new file mode 100644 index 000000000..459fc7853 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_dto.py @@ -0,0 +1,23 @@ +from enum import unique, Enum + +from cc_python_parser.persistence.documented_dto import DocumentedDTO +from cc_python_parser.persistence.file_content_dto import FileContentDTO + + +@unique +class ParseStatus(Enum): + PSNONE = 0 + PSPARTIALLYPARSED = 1 + PSFULLYPARSED = 2 + + +class FileDTO(DocumentedDTO): + def __init__(self): + super().__init__("") + self.type: str = 'Python' # dir, unknown, binary? + self.path: str = "" + self.file_name: str = "" + self.timestamp = "" + self.content: FileContentDTO = FileContentDTO("") + self.parent: str = "" + self.parse_status: int = 0 # PSNONE diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py new file mode 100644 index 000000000..6a374a823 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/file_position_dto.py @@ -0,0 +1,16 @@ +from cc_python_parser.common.position import Range + + +class FilePositionDTO: + def __init__(self, file: str, r: Range): + self.file: str = file + self.range: Range = r + + def __hash__(self): + return hash(str(self.file)) ^ hash(self.range) + + def __eq__(self, other): + return self.file == other.file and self.range == other.range + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py new file mode 100644 index 000000000..0e51ea271 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/function_dto.py @@ -0,0 +1,14 @@ +from typing import Set, List + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.base_dto import DeclarationDTO, UsageDTO +from cc_python_parser.persistence.documented_dto import DocumentedDTO + + +class FunctionDeclarationDTO(DeclarationDTO, DocumentedDTO): + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], + usages: List[UsageDTO], documentation: str, params: List[str], local_vars: List[str]): + DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) + DocumentedDTO.__init__(self, documentation) + self.parameters: List[str] = params + self.locals: List[str] = local_vars diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py new file mode 100644 index 000000000..5dcf51444 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py @@ -0,0 +1,37 @@ +from pathlib import PurePath +from typing import Set, Dict, List + +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.persistence.base_dto import create_file_position_dto +from cc_python_parser.persistence.file_position_dto import FilePositionDTO + + +class ImportDataDTO: + def __init__(self, imported: PurePath, pos: FilePosition): + self.imported: str = str(imported) + self.position: FilePositionDTO = create_file_position_dto(pos) + + def __hash__(self): + return hash(self.imported) ^ hash(self.position) + + def __eq__(self, other): + return self.imported == other.imported and self.position == other.position + + def __ne__(self, other): + return not self.__eq__(other) + + +class ImportDTO: + def __init__(self, importer: PurePath): + self.importer: str = str(importer) + self.imported_modules: List[ImportDataDTO] = [] + self.imported_symbols: Dict[ImportDataDTO, List[str]] = {} + + def add_module_import(self, imported: PurePath, pos: FilePosition): + self.imported_modules.append(ImportDataDTO(imported, pos)) + + def add_symbol_import(self, imported: PurePath, pos: FilePosition, symbol_qualified_name: str): + imported_data_dto = ImportDataDTO(imported, pos) + if imported_data_dto not in self.imported_symbols: + self.imported_symbols[imported_data_dto] = [] + self.imported_symbols[imported_data_dto].append(symbol_qualified_name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py new file mode 100644 index 000000000..55dc77448 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py @@ -0,0 +1,77 @@ +from pathlib import PurePath + +from cc_python_parser.persistence.file_dto import FileDTO +from cc_python_parser.persistence.variable_dto import VariableDeclarationDTO +from cc_python_parser.persistence.function_dto import FunctionDeclarationDTO +from cc_python_parser.persistence.class_dto import ClassDeclarationDTO +from cc_python_parser.persistence.import_dto import ImportDTO + + +class ModelPersistence: + def __init__(self, c_persistence): + self.c_persistence = c_persistence + self.check_c_persistence() + self.log = open(str(PurePath(__file__).parent.joinpath('persistence_log.txt')), 'w+') + + def __del__(self): + self.log.close() + + def check_c_persistence(self): + if self.c_persistence is None: + return + assert hasattr(self.c_persistence, 'persist_file') + assert hasattr(self.c_persistence, 'persist_variable') + assert hasattr(self.c_persistence, 'persist_function') + assert hasattr(self.c_persistence, 'persist_preprocessed_class') + assert hasattr(self.c_persistence, 'persist_class') + assert hasattr(self.c_persistence, 'persist_import') + + def persist_file(self, file: FileDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.print(f"Persist file: {file.path}") + self.c_persistence.persist_file(file) + else: + self.log.write(f"Persist file: {file.path}\n") + + def persist_variable(self, declaration: VariableDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.print(f"Persist var: {declaration.qualified_name}") + self.c_persistence.persist_variable(declaration) + else: + self.log.write(f"Persist var: {declaration.qualified_name}\n") + + def persist_function(self, declaration: FunctionDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.print(f"Persist func: {declaration.qualified_name}") + self.c_persistence.persist_function(declaration) + else: + self.log.write(f"Persist func: {declaration.qualified_name}\n") + + def persist_preprocessed_class(self, declaration: ClassDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.print(f"Persist preprocessed class: {declaration.qualified_name}") + self.c_persistence.persist_preprocessed_class(declaration) + else: + self.log.write(f"Persist preprocessed class: {declaration.qualified_name}\n") + + def persist_class(self, declaration: ClassDeclarationDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.print(f"Persist class: {declaration.qualified_name}") + self.c_persistence.persist_class(declaration) + else: + self.log.write(f"Persist class: {declaration.qualified_name}\n") + + def persist_import(self, imports: ImportDTO) -> None: + if self.c_persistence is not None: + self.c_persistence.print("Persist import") + self.c_persistence.persist_import(imports) + else: + self.log.write("Persist import\n") + + +model_persistence = ModelPersistence(None) + + +def init_persistence(c_persistence): + global model_persistence + model_persistence = ModelPersistence(c_persistence) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py new file mode 100644 index 000000000..fcbb67938 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/variable_dto.py @@ -0,0 +1,5 @@ +from cc_python_parser.persistence.base_dto import DeclarationDTO + + +class VariableDeclarationDTO(DeclarationDTO): + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py b/plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py new file mode 100644 index 000000000..aca4b859d --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/placeholder_function_declaration_cache.py @@ -0,0 +1,52 @@ +import ast +from typing import Dict, Tuple, Optional, Set, Union, List + +from cc_python_parser.common.hashable_list import OrderedHashableList +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.type_data import DeclarationType + + +# TODO: redefined functions? +class PlaceholderFunctionDeclarationCache: + class FunctionDeclarationData: + def __init__(self, func: Union[ast.FunctionDef, ast.AsyncFunctionDef], is_method: bool): + self.function = func + self.is_method = is_method + + def __init__(self): + self.cache: \ + Dict[FunctionDeclaration, + Tuple[PlaceholderFunctionDeclarationCache.FunctionDeclarationData, + Dict[OrderedHashableList[DeclarationType], Set[DeclarationType]]]] = {} + + def add_function_declaration(self, declaration: FunctionDeclaration, function_def: FunctionDeclarationData) -> None: + self.cache[declaration] = (function_def, {}) + + def get_function_def(self, declaration: FunctionDeclaration) -> Optional[FunctionDeclarationData]: + if declaration in self.cache: + return self.cache[declaration][0] + return None + + def get_function_return_type(self, declaration: FunctionDeclaration, + param_types: OrderedHashableList[DeclarationType]) -> Optional[Set[DeclarationType]]: + if declaration in self.cache and param_types in self.cache[declaration][1]: + return self.cache[declaration][1][param_types] + return None + + def get_functions_return_type(self, declarations: Set[FunctionDeclaration], + param_types: OrderedHashableList[DeclarationType]) \ + -> Set[DeclarationType]: + types = set() + for declaration in declarations: + t = self.get_function_return_type(declaration, param_types) + if t is not None: + types.update(t) + return types + + def add_function_return_type(self, declaration: FunctionDeclaration, + param_types: OrderedHashableList[DeclarationType], return_type: Set[DeclarationType]) \ + -> None: + if declaration in self.cache and param_types not in self.cache[declaration][1]: + self.cache[declaration][1][param_types] = return_type + elif declaration not in self.cache: + assert False, "Key not found" diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py new file mode 100644 index 000000000..d28bb7895 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_data.py @@ -0,0 +1,7 @@ +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition + + +class PreprocessedDeclaration(Declaration): + def __init__(self, name: str, file_position: FilePosition): + Declaration.__init__(self, name, "", file_position) # TODO: need qualified name? diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py new file mode 100644 index 000000000..0bf192797 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_file.py @@ -0,0 +1,73 @@ +import ast +from pathlib import PurePath +from typing import Set, Any, Optional + +from cc_python_parser.class_preprocessor import PreprocessedClassCollector +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.import_preprocessor import ImportTable +from cc_python_parser.preprocessed_function import PreprocessedFunction +from cc_python_parser.preprocessed_variable import PreprocessedVariable + + +class PreprocessedFile(ast.NodeVisitor): + def __init__(self, path: Optional[PurePath]): + self.path = path + self.class_collector = PreprocessedClassCollector(path) + self.preprocessed_functions: Set[PreprocessedFunction] = set() + self.preprocessed_variables: Set[PreprocessedVariable] = set() + self.import_table = ImportTable() + self.depth = 0 + self.documentation: str = "" + + def visit_Module(self, node: ast.Module) -> Any: + documentation = ast.get_docstring(node) + if documentation is not None: + self.documentation = documentation + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> Any: + is_local = False + if self.depth > 0: + is_local = True + self.import_table.append_import(node, is_local) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: + is_local = False + if self.depth > 0: + is_local = True + self.import_table.append_import_from(node, is_local) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> Any: + self.class_collector.append_class(node) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + self.class_collector.class_processed() + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + if self.depth == 0: + self.preprocessed_functions.add( + PreprocessedFunction(node.name, FilePosition(self.path, create_range_from_ast_node(node)), node)) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: + if self.depth == 0: + self.preprocessed_functions.add( + PreprocessedFunction(node.name, FilePosition(self.path, create_range_from_ast_node(node)), node)) + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_Assign(self, node: ast.Assign) -> Any: + if self.depth == 0: + for target in node.targets: + if isinstance(target, ast.Name): + self.preprocessed_variables.add( + PreprocessedVariable(target.id, FilePosition(self.path, create_range_from_ast_node(node)))) + else: + pass # print("...") diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py new file mode 100644 index 000000000..317ad0fca --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_function.py @@ -0,0 +1,22 @@ +import ast +from typing import List, Union + +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.preprocessed_data import PreprocessedDeclaration + + +class PreprocessedFunction(PreprocessedDeclaration): + def __init__(self, name: str, file_position: FilePosition, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]): + super().__init__(name, file_position) + self.type_usages: List[Declaration] = [] + self.node: Union[ast.FunctionDef, ast.AsyncFunctionDef] = node + + def __eq__(self, other): + return isinstance(other, PreprocessedFunction) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py new file mode 100644 index 000000000..d120e32f6 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/preprocessed_variable.py @@ -0,0 +1,20 @@ +from typing import List + +from cc_python_parser.base_data import Declaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.preprocessed_data import PreprocessedDeclaration + + +class PreprocessedVariable(PreprocessedDeclaration): + def __init__(self, name: str, file_position: FilePosition): + super().__init__(name, file_position) + self.type_usages: List[Declaration] = [] + + def __eq__(self, other): + return isinstance(other, PreprocessedVariable) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) ^ hash(self.name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py b/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py new file mode 100644 index 000000000..e9b18a037 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py @@ -0,0 +1,26 @@ +import os +from pathlib import PurePath + +from cc_python_parser.parse_exception import ParseException +from cc_python_parser.parser import Parser +from cc_python_parser.persistence.persistence import init_persistence, ModelPersistence + + +def parse(source_root: str, persistence): + init_persistence(persistence) + + def directory_exception(path: PurePath) -> bool: + directory = os.path.basename(os.path.normpath(str(path))) + return directory.startswith('.') or directory == 'venv' + + def file_exception(path: PurePath) -> bool: + return False + + exception = ParseException(directory_exception, file_exception) + p = Parser([source_root], ModelPersistence(persistence), exception) + p.parse() + pass + + +if __name__ == '__main__': + parse("dummy", None) # error! diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope.py b/plugins/python/parser/src/scripts/cc_python_parser/scope.py new file mode 100644 index 000000000..6c093fa3e --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope.py @@ -0,0 +1,223 @@ +from typing import List, Optional +from abc import abstractmethod, ABC +from uuid import uuid4 + +from cc_python_parser.base_data import Declaration, Usage +from cc_python_parser.built_in_types import String, Dictionary +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.import_preprocessor import ImportTable +from cc_python_parser.logger import logger +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.type_data import InitVariablePlaceholderType +from cc_python_parser.variable_data import VariableDeclaration + + +def create_qualified_name(scope_qualified_name: str, current_scope: Optional[str], symbol_name: str, line: int): + qualified_name = scope_qualified_name + if current_scope is not None: + qualified_name = qualified_name + '.' + current_scope + qualified_name = qualified_name + '.' + symbol_name + ':' + str(line) + return qualified_name + + +class Scope(ABC): + def __init__(self, name: str): + self.name: str = name + self.variable_declarations: List[VariableDeclaration] = [] + self.function_declarations: List[FunctionDeclaration] = [] + self.class_declarations: List[ClassDeclaration] = [] + # TODO: should use only self.declarations, and filter the others? (slower, but less memory) + self.declarations: List[Declaration] = [] + + def append_variable(self, variable_declaration: VariableDeclaration) -> None: + self.variable_declarations.append(variable_declaration) + self.declarations.append(variable_declaration) + logger.debug('Var: ' + variable_declaration.name + ' (' + str(variable_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ' + variable_declaration.get_type_repr()) + + def append_function(self, function_declaration: FunctionDeclaration) -> None: + self.function_declarations.append(function_declaration) + self.declarations.append(function_declaration) + logger.debug('Func: ' + function_declaration.name + ' (' + str(function_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ' + function_declaration.get_type_repr()) + + def append_class(self, class_declaration: ClassDeclaration) -> None: + self.class_declarations.append(class_declaration) + self.declarations.append(class_declaration) + logger.debug('Class: ' + class_declaration.name + ' (' + str(class_declaration.file_position) + ') {' + + self.get_type() + ' - ' + self.name + '} ') + + def get_variable(self, usage: Usage) -> Optional[VariableDeclaration]: + return self.get_variable_by_name(usage.name) + + def get_variable_by_name(self, name: str) -> Optional[VariableDeclaration]: + variables = [x for x in self.variable_declarations if x.name == name] + # TODO: fix this assert + # assert len(variables) <= 1 + if len(variables) > 0: + return variables[0] + else: + return None + + def get_function(self, usage: Usage) -> Optional[FunctionDeclaration]: + return self.get_function_by_name(usage.name) + + def get_function_by_name(self, name: str) -> Optional[FunctionDeclaration]: + functions = [x for x in self.function_declarations if x.name == name] + if len(functions) > 0: + return functions[-1] + else: + return None + + def get_class_by_name(self, name: str) -> Optional[ClassDeclaration]: + classes = [x for x in self.class_declarations if x.name == name] + if len(classes) > 0: + return classes[-1] + else: + return None + + @abstractmethod + def get_type(self) -> str: + pass + + +class LifetimeScope(Scope, ABC): + """Symbols declared inside this scope are deleted after exiting from it.""" + + def __init__(self, name: str): + super().__init__(name) + + +class ImportScope(ABC): + """Imported modules are available during this scope.""" + + def __init__(self): + self.import_table: ImportTable = ImportTable() + + +class GlobalScope(LifetimeScope, ImportScope): + """The first scope, which lives on with its module.""" + + def __init__(self, qualified_scope_name: str, file_position: FilePosition): + LifetimeScope.__init__(self, '') + ImportScope.__init__(self) + self.append_variable(VariableDeclaration('__dict__', + create_qualified_name(qualified_scope_name, None, '__dict__', + file_position.range.start_position.line), + file_position, {Dictionary()})) + self.append_variable(VariableDeclaration('__doc__', + create_qualified_name(qualified_scope_name, None, '__doc__', + file_position.range.start_position.line), + file_position, {String()})) + self.append_variable(VariableDeclaration('__module__', + create_qualified_name(qualified_scope_name, None, '__module__', + file_position.range.start_position.line), + file_position, {String()})) + + def get_type(self) -> str: + return "global" + + +class FunctionScope(LifetimeScope, ImportScope): + """Scope of the functions and methods.""" + + def __init__(self, name: str, qualified_scope_name: str, file_position: FilePosition): + LifetimeScope.__init__(self, name) + ImportScope.__init__(self) + self.append_variable(VariableDeclaration('__dict__', + create_qualified_name(qualified_scope_name, name, '__dict__', + file_position.range.start_position.line), + file_position, {Dictionary()})) + self.append_variable(VariableDeclaration('__doc__', + create_qualified_name(qualified_scope_name, name, '__doc__', + file_position.range.start_position.line), + file_position, {String()})) + self.append_variable(VariableDeclaration('__module__', + create_qualified_name(qualified_scope_name, name, '__module__', + file_position.range.start_position.line), + file_position, {String()})) + + def get_type(self) -> str: + return "function" + + +class ClassScope(LifetimeScope, ImportScope): + """Scope of the classes.""" + + def __init__(self, name: str, qualified_scope_name: str, file_position: FilePosition): + LifetimeScope.__init__(self, name) + ImportScope.__init__(self) + self.init_placeholders: List[InitVariablePlaceholderType] = [] + self.append_variable(VariableDeclaration('__dict__', + create_qualified_name(qualified_scope_name, name, '__dict__', + file_position.range.start_position.line), + file_position, {Dictionary()})) + self.append_variable(VariableDeclaration('__doc__', + create_qualified_name(qualified_scope_name, name, '__doc__', + file_position.range.start_position.line), + file_position, {String()})) + self.append_variable(VariableDeclaration('__module__', + create_qualified_name(qualified_scope_name, name, '__module__', + file_position.range.start_position.line), + file_position, {String()})) + + def get_type(self) -> str: + return "class" + + +class PartialLifetimeScope(LifetimeScope, ABC): + """Not all of the symbols are deleted after existing from it.""" + pass + + +class ExceptionScope(PartialLifetimeScope): + """The exception variable is deleted after the 'except' scope, + but all the other symbols declared in it, are alive after it.""" + + def __init__(self): + super().__init__('exception') + + def get_type(self) -> str: + return 'exception' + + +class LambdaScope(LifetimeScope): + """Scope of lambdas.""" + + def __init__(self): + super().__init__('lambda_' + uuid4().hex) + + def get_type(self) -> str: + return "lambda" + + +class GeneratorScope(LifetimeScope): + """Scope of generators.""" + + def __init__(self): + super().__init__('generator_' + uuid4().hex) + + def get_type(self) -> str: + return "generator" + + +class ConditionalScope(Scope): + """Scope of if.""" + + def __init__(self): + super().__init__('if') + + def get_type(self) -> str: + return "conditional" + + +# TODO: need WhileScope and ForScope? +class LoopScope(Scope): + """Scope of for and while.""" + + def __init__(self): + super().__init__('loop') + + def get_type(self) -> str: + return "loop" diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py new file mode 100644 index 000000000..46f4e7add --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py @@ -0,0 +1,499 @@ +import ast +from pathlib import PurePath, Path +from typing import List, Type, Iterator, Callable, Optional, Union + +from cc_python_parser.base_data import Declaration, ImportedDeclaration +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.class_init_declaration import ClassInitDeclaration +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.import_preprocessor import ImportTable, Import +from cc_python_parser.member_access_collector import MemberAccessCollectorIterator, MemberAccessCollector +from cc_python_parser.persistence.import_dto import ImportDTO +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.placeholder_function_declaration_cache import PlaceholderFunctionDeclarationCache +from cc_python_parser.scope import Scope, ClassScope, GlobalScope, FunctionScope, LifetimeScope, PartialLifetimeScope, \ + ConditionalScope, LoopScope, ImportScope +from cc_python_parser.type_data import PlaceholderType +from cc_python_parser.variable_data import VariableDeclaration, ModuleVariableDeclaration, FunctionVariableDeclaration, \ + TypeVariableDeclaration +from cc_python_parser.common.position import Range +from cc_python_parser.common.file_position import FilePosition + + +class ScopeManager: + def __init__(self, current_file: PurePath, import_finder: ImportFinder, persistence: ModelPersistence, + scopes: Optional[List[Scope]] = None): + self.current_file = current_file + self.import_finder = import_finder + if scopes is None: + self.scopes: List[Scope] = [GlobalScope(self.get_qualified_scope_name_from_current_scope([]), + FilePosition(current_file, Range.get_empty_range()))] + else: + self.scopes = scopes + self.persistence: ModelPersistence = persistence + self.placeholder_function_declaration_cache = PlaceholderFunctionDeclarationCache() + self.persist_symbols: int = 0 + + def lock_persisting(self): + self.persist_symbols += 1 + + def unlock_persisting(self): + self.persist_symbols -= 1 + + def is_current_scope_instance(self, t: Type) -> bool: + return isinstance(self.scopes[-1], t) + + # TODO: local functions + def is_current_scope_method(self) -> bool: + for i in range(len(self.scopes) - 1, -1, -1): + if isinstance(self.scopes[i], LifetimeScope) and not isinstance(self.scopes[i], PartialLifetimeScope): + return isinstance(self.scopes[i], FunctionScope) and isinstance(self.scopes[i - 1], ClassScope) + return False + + def is_current_scope_init(self) -> bool: + return self.is_current_scope_method() and self.get_current_lifetime_scope().name == '__init__' + + def get_current_lifetime_scope(self) -> LifetimeScope: + for scope in self.reverse(): + if isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): + return scope + assert False # there should be always a GlobalScope, which is LifetimeScope + + def reverse(self) -> Iterator[Scope]: + for scope in reversed(self.scopes): + yield scope + + def iterate(self) -> Iterator[Scope]: + for scope in self.scopes: + yield scope + + def with_scope(self, scope: Scope, action: Callable[[], None]) -> None: + self.scopes.append(scope) + action() + self.persist_current_scope() + del self.scopes[-1] + + def get_size(self) -> int: + return len(self.scopes) + + def get_current_scope(self) -> Scope: + return self.scopes[-1] + + def get_global_scope(self) -> GlobalScope: + assert len(self.scopes) > 0 + global_scope = self.scopes[0] + if not isinstance(global_scope, GlobalScope): + assert False + return global_scope + + def append_variable_to_current_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_lifetime_scope().append_variable(new_variable) + + def append_exception_to_current_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_partial_lifetime_scope().append_variable(new_variable) + + def append_function_to_current_scope(self, new_function: FunctionDeclaration) -> None: + self.get_current_lifetime_scope().append_function(new_function) + + def append_class_to_current_scope(self, new_class: ClassDeclaration) -> None: + self.get_current_lifetime_scope().append_class(new_class) + + def append_variable_to_current_class_scope(self, new_variable: VariableDeclaration) -> None: + self.get_current_class_scope().append_variable(new_variable) + + def append_function_to_current_class_scope(self, new_function: FunctionDeclaration) -> None: + self.get_current_class_scope().append_function(new_function) + + def append_class_to_current_class_scope(self, new_class: ClassDeclaration) -> None: + self.get_current_class_scope().append_class(new_class) + + def append_module_variable_to_current_scope(self, module_variable: ModuleVariableDeclaration) -> None: + def is_same_declaration(var: VariableDeclaration) -> bool: + return isinstance(var, ModuleVariableDeclaration) and \ + var.imported_module_location == module_variable.imported_module_location + scope = self.get_current_lifetime_scope() + if len([v for v in scope.variable_declarations if is_same_declaration(v)]) == 0: + scope.append_variable(module_variable) + + def get_current_partial_lifetime_scope(self) -> Optional[LifetimeScope]: + for scope in self.reverse(): + if isinstance(scope, PartialLifetimeScope): + return scope + return None + + def get_current_class_scope(self) -> Optional[ClassScope]: + for scope in self.reverse(): + if isinstance(scope, ClassScope): + return scope + return None + + def get_current_scope_name(self) -> str: + return self.scopes[-1].name + + def is_inside_conditional_scope(self) -> bool: + for scope in self.reverse(): + if isinstance(scope, (ConditionalScope, LoopScope)): + return True + elif isinstance(scope, LifetimeScope) and not isinstance(scope, PartialLifetimeScope): + return False + assert False # GlobalScope is LifetimeScope + + def is_declaration_in_current_scope(self, name: str) -> bool: + return self.is_declaration_in_scope(name, self.scopes[-1]) + + def is_declaration_in_current_lifetime_scope(self, name: str) -> bool: + return self.is_declaration_in_scope(name, self.get_current_lifetime_scope()) + + def is_declaration_in_current_class(self, name: str) -> bool: + class_scope = self.get_current_class_scope() + if class_scope is not None: + return self.is_declaration_in_scope(name, class_scope) + return False + + def get_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.get_current_lifetime_scope(): + continue # members can be accessed via self, except from the current class scope + declaration = self.get_declaration_from_scope(name, scope) + if declaration is not None: + from cc_python_parser.variable_data import TypeVariableDeclaration + if isinstance(declaration, TypeVariableDeclaration): + pass # print("NEED FIX: TypeVariable") + return declaration + current_class_scope = self.get_current_class_scope() + if current_class_scope is not None: + for init_placeholder in current_class_scope.init_placeholders: + if name == init_placeholder.name: + return init_placeholder + return PlaceholderType(name) + + def get_variable_declaration(self, name: str) -> Union[VariableDeclaration, PlaceholderType]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.scopes[-1]: + continue + declaration = self.get_variable_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_function_declaration(self, name: str) -> Union[Declaration, PlaceholderType]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.scopes[-1]: + continue + declaration = self.get_function_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_class_declaration(self, name: str) -> Union[ClassDeclaration, PlaceholderType]: + for scope in self.reverse(): + if isinstance(scope, ClassScope) and scope is not self.scopes[-1]: + continue + declaration = self.get_class_declaration_from_scope(name, scope) + if declaration is not None: + return declaration + return PlaceholderType(name) + + def get_declaration_from_member_access(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: + iter(iterator) + members = [next(iterator)] + # TODO: MethodVariable? + if isinstance(members[0], MemberAccessCollector.MethodData): + declaration = self.get_function_declaration(members[0].name) + elif isinstance(members[0], MemberAccessCollector.AttributeData): + declaration = self.get_variable_declaration(members[0].name) + else: + declaration = self.get_declaration(members[0].name) + if not isinstance(declaration, PlaceholderType): + return declaration + try: + module_variable = self.get_imported_module(iterator) + if module_variable is not None: + declaration = module_variable + else: + declaration_from_other_module = self.get_declaration_from_other_module(iterator) + if declaration_from_other_module is not None: + declaration = declaration_from_other_module + except StopIteration: + assert False + return declaration + + def get_scope_manager_from_member_access(self, iterator: MemberAccessCollectorIterator) -> Optional['ScopeManager']: + iter(iterator) + members = [next(iterator)] + if isinstance(members[0], MemberAccessCollector.MethodData): + declaration = self.get_function_declaration(members[0].name) + else: + declaration = self.get_declaration(members[0].name) + if not isinstance(declaration, PlaceholderType): + if isinstance(declaration, ImportedDeclaration): + return self.import_finder.get_scope_manager_by_location(declaration.module) + else: + return self + try: + module_import = self.get_import(iterator) + if module_import is not None: + return self.import_finder.get_scope_manager_by_location(module_import.location) + except StopIteration: + assert False + return None + + # import a.b.c -> c.f() vs a.b.c.f() + def get_declaration_from_other_module(self, iterator: MemberAccessCollectorIterator) -> Optional[Declaration]: + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + # TODO: is this alias check enough? + if module.module.alias is None and len(module.path) >= len(iterator.mac.call_list): + continue + is_same = True + current = next(iterator) + name = current.name + iter(iterator) + if module.module.alias is not None: + is_same = module.module.alias == name + next(iterator) + else: + for i in range(0, len(module.path)): + name = next(iterator).name + if module.path[i] != name: + is_same = False + break + if is_same: + name = next(iterator).name + if module.is_module_import(): + other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) + if other_scope_manager is not None: + if isinstance(current, MemberAccessCollector.MemberData): + return other_scope_manager.get_function_declaration(name) + else: + return other_scope_manager.get_declaration(name) + for imp in module.imported: + if (imp.alias is not None and imp.alias == name) \ + or imp.name == name or imp.is_all_imported(): + other_scope_manager = self.import_finder.get_scope_manager_by_location(module.location) + if other_scope_manager is not None: + if isinstance(current, MemberAccessCollector.MemberData): + return other_scope_manager.get_function_declaration(name) + else: + return other_scope_manager.get_declaration(name) + return None + + def get_imported_module(self, iterator: MemberAccessCollectorIterator) -> Optional[ModuleVariableDeclaration]: + imported_module = None + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + if module.module.alias is None and len(module.path) != len(iterator.mac.call_list): + continue + if module.module.alias is not None: + if module.module.alias == iterator.mac.call_list[-1].name: + imported_module = module + break + else: + continue + else: + for i in range(0, len(module.path)): + next(iterator) + module_path = [m.name for m in iterator.mac.call_list] + module_path.reverse() + if module.path == module_path: + imported_module = module + break + if imported_module is not None and isinstance(scope, Scope): + for var in reversed(scope.variable_declarations): + if isinstance(var, ModuleVariableDeclaration) and \ + var.imported_module_location == imported_module.location: + return var + return None + + def get_import(self, iterator: MemberAccessCollectorIterator) -> Optional[Import]: + for scope in self.reverse(): + if isinstance(scope, ImportScope): + for module in reversed(scope.import_table.modules): + iter(iterator) + # TODO: is this alias check enough? + if module.module.alias is None and len(module.path) >= len(iterator.mac.call_list): + continue + current = next(iterator) + name = current.name + iter(iterator) + if module.module.alias is not None: + if module.module.alias == name: + return module + else: + for i in range(0, len(module.path)): + name = next(iterator).name + if module.path[i] != name: + return module + return None + + def get_declaration_from_current_class(self, name: str) -> Union[Declaration, PlaceholderType]: + declaration = self.get_declaration_from_scope(name, self.get_current_class_scope()) + if declaration is None: + return PlaceholderType(name) + return declaration + + def get_nonlocal_variable(self, name: str) -> Optional[VariableDeclaration]: + assert len(self.scopes) > 2 + for scope in self.scopes[-2:0:-1]: + var = scope.get_variable_by_name(name) + if var is not None: + return var + return None + + def get_global_variable(self, name: str) -> Optional[VariableDeclaration]: + return self.scopes[0].get_variable_by_name(name) + + @staticmethod + def get_declaration_from_scope(name: str, scope: Scope) -> Optional[Declaration]: + for declaration in reversed(scope.declarations): + if declaration.name == name: + return declaration + return None + + @staticmethod + def is_declaration_in_scope(name: str, scope: Scope) -> bool: + for var in scope.declarations: + if var.name == name: + return True + return False + + @staticmethod + def get_variable_declaration_from_scope(name: str, scope: Scope) -> Optional[VariableDeclaration]: + for declaration in reversed(scope.declarations): + if declaration.name != name: + continue + elif isinstance(declaration, VariableDeclaration): + return declaration + elif isinstance(declaration, FunctionDeclaration) and not isinstance(declaration, ClassInitDeclaration): + return FunctionVariableDeclaration(declaration) + elif isinstance(declaration, ClassDeclaration): + return TypeVariableDeclaration(declaration) + return None + + @staticmethod + def get_function_declaration_from_scope(name: str, scope: Scope) \ + -> Optional[Union[FunctionDeclaration, ClassDeclaration, FunctionVariableDeclaration]]: + for declaration in reversed(scope.declarations): + if declaration.name != name: + continue + elif isinstance(declaration, (FunctionDeclaration, ClassDeclaration)): + return declaration + elif isinstance(declaration, ClassDeclaration): + return ClassInitDeclaration(declaration) + else: + function_variables = [t for t in declaration.type if isinstance(t, FunctionVariableDeclaration)] + if len(function_variables) > 0: + return function_variables[0] + else: + return declaration + return None + + @staticmethod + def get_class_declaration_from_scope(name: str, scope: Scope) -> Optional[ClassDeclaration]: + for declaration in reversed(scope.class_declarations): + if declaration.name == name: + return declaration + return None + + def get_current_class_declaration(self) -> Optional[ClassDeclaration]: + current_class_scope_found = False + for scope in self.reverse(): + if current_class_scope_found and isinstance(scope, LifetimeScope): + return scope.class_declarations[-1] + elif not current_class_scope_found: + current_class_scope_found = isinstance(scope, ClassScope) + return None + + def set_global_import_table(self, import_table: ImportTable): + global_scope = self.scopes[0] + if isinstance(global_scope, GlobalScope): + global_scope.import_table = import_table + + def append_import(self, node: ast.Import): + for scope in self.reverse(): + if isinstance(scope, ImportScope): + scope.import_table.append_import(node, not isinstance(scope, GlobalScope)) + return + assert False + + def append_import_from(self, node: ast.ImportFrom): + for scope in self.reverse(): + if isinstance(scope, ImportScope): + scope.import_table.append_import_from(node, not isinstance(scope, GlobalScope)) + return + assert False + + def handle_import_scope(self): + if isinstance(self.scopes[-1], ImportScope): + for declaration in self.scopes[-1].declarations: + if isinstance(declaration, ImportedDeclaration): + declaration.imported_declaration.usages.extend(declaration.usages) + + def persist_current_scope(self): + # for declaration in self.scopes[-1].declarations: + # print(self.get_qualified_name_from_current_scope(declaration)) + + if self.persist_symbols > 0: + return + + self.handle_import_scope() + + import_dto = ImportDTO(self.current_file) + + for var in self.scopes[-1].variable_declarations: + if isinstance(var, ImportedDeclaration): + import_dto.add_symbol_import(var.module.location, var.position, + var.imported_declaration.qualified_name) + elif isinstance(var, ModuleVariableDeclaration): + import_dto.add_module_import(var.imported_module_location, var.file_position) + else: + self.persistence.persist_variable(var.create_dto()) + + for func in self.scopes[-1].function_declarations: + if isinstance(func, ImportedDeclaration): + import_dto.add_symbol_import(func.module.location, func.position, + func.imported_declaration.qualified_name) + else: + self.persistence.persist_function(func.create_dto()) + + for cl in self.scopes[-1].class_declarations: + if isinstance(cl, ImportedDeclaration): + import_dto.add_symbol_import(cl.module.location, cl.position, + cl.imported_declaration.qualified_name) + else: + self.persistence.persist_class(cl.create_dto()) + + if len(import_dto.imported_modules) > 0 or len(import_dto.imported_symbols) > 0: + self.persistence.persist_import(import_dto) + + def get_qualified_scope_name_from_current_scope(self, scopes: Optional[List[Scope]] = None) -> str: + qualified_name_parts = [] + if scopes is None: + for scope in reversed(self.scopes): + if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope + qualified_name_parts.append(scope.name) + else: + for scope in reversed(scopes): + if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope + qualified_name_parts.append(scope.name) + file_name = self.current_file.name + assert file_name[-3:] == '.py' + qualified_name_parts.append(file_name[:-3]) + directory = self.current_file.parent + while True: + init_file_location = Path(directory).joinpath("__init__.py") + if init_file_location.exists(): + qualified_name_parts.append(directory.name) + directory = directory.parent + else: + qualified_name_parts.append(str(hash(directory))) + break + return '.'.join(reversed(qualified_name_parts)) + + def get_qualified_name_from_current_scope(self, declaration_name: str, line_num: int) -> str: + return self.get_qualified_scope_name_from_current_scope() + '.' + declaration_name + ':' + str(line_num) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py new file mode 100644 index 000000000..b73af54b2 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py @@ -0,0 +1,1115 @@ +import ast +from pathlib import PurePath +from typing import List, Optional, Any, Union, Set, TypeVar + +from cc_python_parser.built_in_functions import get_built_in_function +from cc_python_parser.built_in_types import GenericType, BuiltIn +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.placeholder_function_declaration_cache import PlaceholderFunctionDeclarationCache +from cc_python_parser.symbol_collector_interface import SymbolCollectorBase +from cc_python_parser.base_data import Declaration, Usage, ImportedDeclaration +from cc_python_parser.class_data import ClassDeclaration, ImportedClassDeclaration +from cc_python_parser.class_preprocessor import PreprocessedClass +from cc_python_parser.common.parser_tree import ParserTree +from cc_python_parser.common.position import Range +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.import_preprocessor import ImportTable +from cc_python_parser.logger import logger +from cc_python_parser.member_access_collector import MemberAccessCollector, MemberAccessCollectorIterator +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.preprocessed_function import PreprocessedFunction +from cc_python_parser.preprocessed_variable import PreprocessedVariable +from cc_python_parser.scope import ClassScope, FunctionScope, GeneratorScope, LambdaScope, \ + ExceptionScope, ConditionalScope, LoopScope, GlobalScope +from cc_python_parser.function_data import FunctionDeclaration, ImportedFunctionDeclaration, \ + StaticFunctionDeclaration +from cc_python_parser.scope_manager import ScopeManager +from cc_python_parser.symbol_finder import SymbolFinder +from cc_python_parser.type_data import PlaceholderType, InitVariablePlaceholderType, VariablePlaceholderType +from cc_python_parser.type_deduction import TypeDeduction +from cc_python_parser.type_hint_data import TypeHintType +from cc_python_parser.variable_data import VariableDeclaration, ImportedVariableDeclaration, \ + ModuleVariableDeclaration, StaticVariableDeclaration, NonlocalVariableDeclaration, GlobalVariableDeclaration, \ + FunctionVariableDeclaration, TypeVariableDeclaration, MethodVariableDeclaration +from cc_python_parser.common.metrics import metrics + + +class SymbolCollector(ast.NodeVisitor, SymbolFinder, SymbolCollectorBase): + def __init__(self, tree: ParserTree, current_file: PurePath, preprocessed_file: PreprocessedFile, + import_finder: ImportFinder, persistence: ModelPersistence, + function_symbol_collector_factory: FunctionSymbolCollectorFactory): + SymbolCollectorBase.__init__(self, preprocessed_file, import_finder) + self.tree = tree + self.current_file = current_file + self.scope_manager: ScopeManager = ScopeManager(current_file, import_finder, persistence) + self.current_function_declaration: List[FunctionDeclaration] = [] + self.current_class_declaration: List[ClassDeclaration] = [] + self.function_symbol_collector_factory = function_symbol_collector_factory + self.type_deduction = TypeDeduction(self, self.scope_manager, + preprocessed_file, self.function_symbol_collector_factory) + + def collect_symbols(self): + self.visit(self.tree.root.node) + self.post_process() + + def generic_visit(self, node: ast.AST) -> Any: + if type(self) is SymbolCollector: + metrics.add_ast_count() + ast.NodeVisitor.generic_visit(self, node) + + def visit_Import(self, node: ast.Import) -> Any: + metrics.add_import_count() + modules = ImportTable.convert_ast_import_to_import(node) + for module in modules: + if not module.is_module_import(): + continue + # scope.imports.append((module, self.import_finder.get_global_scope_by_location(module.location))) + self.scope_manager.append_module_variable_to_current_scope( + ModuleVariableDeclaration(module.path[-1], module.location, + FilePosition(self.current_file, module.range), + self.import_finder.get_global_scope_by_location(module.location))) + self.scope_manager.append_import(node) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: + metrics.add_import_count() + modules = ImportTable.convert_ast_import_from_to_import(node) + for module in modules: + self.scope_manager.append_module_variable_to_current_scope( + ModuleVariableDeclaration(module.path[-1], module.location, + FilePosition(self.current_file, module.range), + self.import_finder.get_global_scope_by_location(module.location))) + if not module.is_module_import(): + scope_manager: ScopeManager = self.import_finder.get_scope_manager_by_location(module.location) + if scope_manager is None: + # import from library + continue + # in case of circular import this case is not valid (runtime error) -> must find declaration + file_position = FilePosition(self.current_file, create_range_from_ast_node(node)) + # TODO: handle transitive imports + for imported in module.imported: + if imported.is_all_imported(): + for declaration in scope_manager.get_global_scope().declarations: + if isinstance(declaration, VariableDeclaration): + var = ImportedVariableDeclaration( + declaration.name, file_position, declaration, module) + self.scope_manager.append_variable_to_current_scope(var) + self.imported_declaration_scope_map[var] = scope_manager + elif isinstance(declaration, FunctionDeclaration): + func = ImportedFunctionDeclaration( + declaration.name, file_position, declaration, module) + self.scope_manager.append_function_to_current_scope(func) + self.imported_declaration_scope_map[func] = scope_manager + elif isinstance(declaration, ClassDeclaration): + c = ImportedClassDeclaration( + declaration.name, file_position, declaration, module) + self.scope_manager.append_class_to_current_scope(c) + self.imported_declaration_scope_map[c] = scope_manager + else: + assert False + else: + # TODO: redefinition? + declaration = scope_manager.get_declaration(imported.name) + if isinstance(declaration, PlaceholderType): + continue + elif isinstance(declaration, VariableDeclaration): + var = ImportedVariableDeclaration( + imported.name if imported.alias is None else imported.alias, + file_position, declaration, module) + self.scope_manager.append_variable_to_current_scope(var) + self.imported_declaration_scope_map[var] = scope_manager + elif isinstance(declaration, FunctionDeclaration): + func = ImportedFunctionDeclaration( + imported.name if imported.alias is None else imported.alias, + file_position, declaration, module) + self.scope_manager.append_function_to_current_scope(func) + self.imported_declaration_scope_map[func] = scope_manager + elif isinstance(declaration, ClassDeclaration): + c = ImportedClassDeclaration( + imported.name if imported.alias is None else imported.alias, + file_position, declaration, module) + self.scope_manager.append_class_to_current_scope(c) + self.imported_declaration_scope_map[c] = scope_manager + else: + assert False + self.scope_manager.append_import_from(node) + self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> Any: + metrics.add_class_count() + base_classes = self.collect_base_classes(node) + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name, r.start_position.line) + new_class = ClassDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), + base_classes, self.get_documentation(node)) + self.class_def_process_derived_members(new_class, base_classes) + # TODO: handle preprocessed classes + self.scope_manager.append_class_to_current_scope(new_class) + self.current_class_declaration.append(new_class) + self.scope_manager.persistence.persist_preprocessed_class(new_class.create_dto()) + + # TODO: check if has at least 1 parameter? (self) + # TODO: range could be the __init__ + # self.scope_manager.append_function_to_current_scope(ClassInitDeclaration(new_class)) + + def action(): + self.generic_visit(node) + scope = self.scope_manager.get_current_scope() + assert isinstance(scope, ClassScope) + for var in scope.variable_declarations: + if isinstance(var, StaticVariableDeclaration) and var not in new_class.static_attributes: + new_class.static_attributes.append(var) + elif isinstance(var, VariableDeclaration) and var not in new_class.attributes: + new_class.attributes.append(var) + for func in scope.function_declarations: + if isinstance(func, StaticFunctionDeclaration): + new_class.static_methods.append(func) + elif isinstance(func, FunctionDeclaration): + new_class.methods.append(func) + else: + assert False + for c in scope.class_declarations: + new_class.classes.append(c) + + self.scope_manager.with_scope(ClassScope(node.name, + self.scope_manager.get_qualified_scope_name_from_current_scope(), + new_class.file_position), action) + self.class_def_post_process(new_class) + del self.current_class_declaration[-1] + + def collect_base_classes(self, node: ast.ClassDef) -> List[ClassDeclaration]: + base_classes = [] + for base_class in node.bases: + base_class_types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(base_class)) + if len(base_class_types) > 1: + continue # TODO: eg: standard Lib > ctypes > test > test_byteswap.py > function 'test_struct_struct' + for base_class_type in base_class_types: + if isinstance(base_class_type, ClassDeclaration): + base_classes.append(base_class_type) + elif isinstance(base_class_type, TypeVariableDeclaration): + if isinstance(base_class_type.reference, ImportedDeclaration): + base_classes.append(base_class_type.reference.imported_declaration) + elif isinstance(base_class_type.reference, ClassDeclaration): + base_classes.append(base_class_type.reference) + else: + pass # print('') + else: + pass # print('') + return base_classes + + def class_def_process_derived_members(self, cl: ClassDeclaration, base_classes: List[ClassDeclaration]): + declaration_type = TypeVar('declaration_type', bound=Declaration) + + def collect_derived_members(members: List[List[declaration_type]]) -> List[declaration_type]: + declarations = [] + for base_class_members in members: + for member in base_class_members: + if all([m.name != member.name for m in declarations]): + declarations.append(member) + return declarations + + cl.attributes.extend(collect_derived_members([d.attributes for d in base_classes])) + cl.static_attributes.extend(collect_derived_members([d.static_attributes for d in base_classes])) + cl.methods.extend(collect_derived_members([d.methods for d in base_classes])) + cl.static_methods.extend(collect_derived_members([d.static_methods for d in base_classes])) + cl.classes.extend(collect_derived_members([d.classes for d in base_classes])) + + def class_def_post_process(self, cl: ClassDeclaration): + def remove_hidden_declarations(declarations: List[Declaration]): + for declaration in reversed(declarations): + hidden_declarations = [d for d in declarations if d.name == declaration.name and d is not declaration] + for hidden_declaration in hidden_declarations: + declarations.remove(hidden_declaration) + + remove_hidden_declarations(cl.attributes) + remove_hidden_declarations(cl.static_attributes) + remove_hidden_declarations(cl.methods) # TODO: singledispatchmethod + remove_hidden_declarations(cl.static_methods) + remove_hidden_declarations(cl.classes) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + self.visit_common_function_def(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: + self.visit_common_function_def(node) + + def visit_common_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> Any: + metrics.add_function_count() + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(node.name, r.start_position.line) + if self.is_static_method(node): + new_function = StaticFunctionDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), + [], self.get_documentation(node)) + else: + new_function = FunctionDeclaration(node.name, qualified_name, + FilePosition(self.current_file, r), [], self.get_documentation(node)) + self.scope_manager.append_function_to_current_scope(new_function) + self.current_function_declaration.append(new_function) + + # if node.name == '__init__' and isinstance(self.scopes[-1], ClassScope): # ctor ~ __init__ + + def action(): + # self.collect_parameters(node) + self.process_function_def(node) + if any(isinstance(x, PlaceholderType) for x in self.current_function_declaration[-1].type): + is_method = self.scope_manager.is_current_scope_method() + func_declaration_data = PlaceholderFunctionDeclarationCache.FunctionDeclarationData(node, is_method) + self.scope_manager.placeholder_function_declaration_cache. \ + add_function_declaration(new_function, func_declaration_data) + + self.scope_manager.with_scope(FunctionScope(node.name, + self.scope_manager.get_qualified_scope_name_from_current_scope(), + new_function.file_position), action) + del self.current_function_declaration[-1] + + def get_documentation(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef]) -> str: + documentation = ast.get_docstring(node) + if documentation is None: + return "" + else: + return documentation + + def process_function_def(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None: + is_init = self.scope_manager.is_current_scope_init() + is_method = self.scope_manager.is_current_scope_method() + + def get_first_param() -> Optional[ast.arg]: + if len(node.args.posonlyargs) > 0: + return node.args.posonlyargs[0] + if len(node.args.args) > 0: + return node.args.args[0] + if node.args.vararg is not None: + return node.args.vararg + if len(node.args.kwonlyargs) > 0: + return node.args.kwonlyargs[0] + if node.args.kwarg is not None: + return node.args.kwarg + return None + + def get_annotation_type(annotation: Optional[ast.expr]) -> Optional[TypeHintType]: + if annotation is None: + return None + + hinted_type = None + if annotation is not None: + if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): + hinted_type = None + else: + t = self.type_deduction.deduct_type(annotation) + current_type = self.type_deduction.get_current_type(t) + assert len(current_type) == 0 or len(current_type) == 1 + if len(current_type) == 1: + tt = list(current_type)[0] + if isinstance(tt, (ClassDeclaration, BuiltIn)): + hinted_type = TypeHintType(tt) + elif isinstance(tt, TypeVariableDeclaration): + if isinstance(tt.reference, (ClassDeclaration, BuiltIn)): + hinted_type = TypeHintType(tt.reference) + elif isinstance(tt.reference, ImportedDeclaration): + hinted_type = TypeHintType(tt.reference.imported_declaration) + else: + hinted_type = None + else: + hinted_type = None + return hinted_type + + def process_arg(param: Optional[ast.arg]): + if param is None: + return + + r = create_range_from_ast_node(param) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(param.arg, r.start_position.line) + new_variable = VariableDeclaration(param.arg, qualified_name, FilePosition(self.current_file, r)) + + hinted_type = get_annotation_type(param.annotation) + + if param.arg == 'self' and is_method and get_first_param() is param: + new_variable.type.add(self.scope_manager.get_current_class_declaration()) + elif is_init: + if hinted_type is not None: + new_variable.type.add(hinted_type) + else: + init_var_placeholder_type = InitVariablePlaceholderType(param.arg) + new_variable.type.add(init_var_placeholder_type) + self.scope_manager.get_current_class_scope().init_placeholders.append(init_var_placeholder_type) + else: + if hinted_type is not None: + new_variable.type.add(hinted_type) + else: + new_variable.type.add(VariablePlaceholderType(param.arg)) + self.scope_manager.append_variable_to_current_scope(new_variable) + self.current_function_declaration[-1].parameters.append(new_variable) + + for arg in node.args.args: # simple arguments + process_arg(arg) + process_arg(node.args.vararg) # *arg + for arg in node.args.kwonlyargs: # arguments between vararg and kwarg + process_arg(arg) + process_arg(node.args.kwarg) # **arg + for default in node.args.defaults: # last N args default value + self.visit(default) + for default in node.args.kw_defaults: # default values of kwonlyargs (all) + if default is not None: + self.visit(default) + for arg in node.args.posonlyargs: # arguments before / "parameter" (after -> args) + process_arg(arg) + + for decorator in node.decorator_list: + self.visit(decorator) + + if node.returns is not None: + self.visit(node.returns) + + for stmt in node.body: + self.visit(stmt) + + self.current_function_declaration[-1].local_variables.extend( + self.scope_manager.get_current_scope().variable_declarations) + + return_type = get_annotation_type(node.returns) + if return_type is not None: + self.current_function_declaration[-1].type.add(return_type) + + def visit_Return(self, node: ast.Return) -> Any: + self.visit_common_return(node) + + def visit_Yield(self, node: ast.Yield) -> Any: + if isinstance(self.scope_manager.get_current_scope(), LambdaScope): + self.generic_visit(node) + else: + self.visit_common_return(node) + + def visit_YieldFrom(self, node: ast.YieldFrom) -> Any: + self.visit_common_return(node) + + def visit_common_return(self, node: (ast.Return, ast.Yield, ast.YieldFrom)): + assert isinstance(self.current_function_declaration[-1], FunctionDeclaration) + if node.value is not None: + types = self.type_deduction.deduct_type(node) + if any(t is self.current_function_declaration[-1] for t in types): + types = {t for t in types if t is not self.current_function_declaration[-1]} + if len(types) == 0: + self.current_function_declaration[-1].type.add(VariablePlaceholderType('')) + else: + self.current_function_declaration[-1].type.update(types) + self.generic_visit(node) + + def visit_NamedExpr(self, node: ast.NamedExpr) -> Any: + self.handle_assignment(node.target, node.value) + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> Any: + for target in node.targets: + self.handle_assignment(target, node.value) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> Any: + self.handle_assignment(node.target, node.value) + self.generic_visit(node) + + def handle_assignment(self, target: ast.AST, value: ast.AST): + if isinstance(target, (ast.Name, ast.Attribute, ast.Subscript, ast.Starred)): + self.handle_single_assignment(target, value) + elif isinstance(target, (ast.List, ast.Tuple)): + if isinstance(value, (ast.List, ast.Tuple)): + assert len(target.elts) == len(value.elts) + for i in range(0, len(target.elts)): + self.handle_assignment(target.elts[i], value.elts[i]) + else: + for t in target.elts: + self.handle_assignment(t, value) + else: + assert False + + def handle_single_assignment(self, target: ast.AST, value: ast.AST): + mac = None + if isinstance(target, (ast.Attribute, ast.Subscript, ast.Starred)): + mac = MemberAccessCollector(target) + name = mac.call_list[0].name + elif isinstance(target, ast.Name): + name = target.id + else: + assert False + + if name == '_': + return # TODO + + if value is None: # variable: type (no value!) + types = set() + else: + types = self.type_deduction.get_current_type(self.type_deduction.deduct_type(value)) + + if isinstance(target, ast.Subscript): + self.handle_subscript_assignment(mac, types) + return + + r = create_range_from_ast_node(target) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + + if self.is_static_variable(target): + new_variable = StaticVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + else: + new_variable = VariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), + self.type_deduction.get_current_type(types)) + + if self.scope_manager.is_current_scope_init() and mac is not None and len(mac.call_list) == 2 and \ + mac.call_list[-1].name == 'self': + if not self.scope_manager.is_declaration_in_current_class(new_variable.name): + self.scope_manager.append_variable_to_current_class_scope(new_variable) + self.scope_manager.get_current_class_declaration().attributes.append(new_variable) + elif isinstance(target, (ast.Name, ast.Starred)): + if self.scope_manager.is_inside_conditional_scope() and \ + not isinstance(declaration := self.scope_manager.get_declaration(name), PlaceholderType): + if isinstance(value, (ast.Name, ast.Call, ast.Attribute)): + d = self.get_declaration(value) + dd = self.get_declaration(target) + if self.has_recursion_in_type(declaration, types): + declaration.type = self.fix_recursion_in_type(declaration, types) + else: + declaration.type.update(new_variable.type) + declaration.type = \ + {t for t in self.type_deduction.get_current_type(declaration.type) + if not isinstance(t, PlaceholderType)} + else: + if isinstance(value, (ast.Name, ast.Call, ast.Attribute)): + d = self.get_declaration(value) + if d is not None: + new_variable.type = \ + {t for t in self.type_deduction.get_current_type(d) if not isinstance(t, PlaceholderType)} + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + pass + + def has_recursion_in_type(self, var: Optional[Union[VariableDeclaration, PlaceholderType]], types: Set): + if var is None or isinstance(var, PlaceholderType): + return False + for t in types: + if var is t: + return True + elif isinstance(t, GenericType) and self.has_recursion_in_type(var, t.types): + return True + return False + + def fix_recursion_in_type(self, var: Optional[Union[VariableDeclaration, PlaceholderType]], types: Set) -> Set: + if var is None or isinstance(var, PlaceholderType): + return set() + fixed_types = set() + for t in types: + if var is t: + if isinstance(var, GenericType): + fixed_types.update(var.types) + else: + fixed_types.update(var.type) + elif isinstance(t, GenericType): + fixed_types.update(self.fix_recursion_in_type(var, t.types)) + return fixed_types + + def handle_subscript_assignment(self, mac: MemberAccessCollector, types: Set): + name = mac.call_list[0].name + if name is not None: + declaration = self.scope_manager.get_variable_declaration(name) + if not isinstance(declaration, PlaceholderType): + if self.has_recursion_in_type(declaration, types): + declaration.types = self.fix_recursion_in_type(declaration, types) + elif generics := [d for d in declaration.type if isinstance(d, GenericType)]: + for generic in generics: + if self.has_recursion_in_type(generic, types): + generic.types = self.fix_recursion_in_type(generic, types) + else: + generic.types.update(types) + elif len(mac.call_list) == 1 and isinstance(mac.call_list[0], MemberAccessCollector.ReturnValueData) and \ + isinstance(mac.call_list[0].node, ast.BoolOp): + pass # TODO: (list_1 or list_2)[i] = x, eg: Lib > idlelib > query.py > showerror function + else: # eg.: a[i][j] = k + depth = 1 + for i in range(1, len(mac.call_list)): + if not isinstance(mac.call_list[i], MemberAccessCollector.ReturnValueData): + depth = i + break + declarations = {self.scope_manager.get_variable_declaration(mac.call_list[depth].name)} + generic_types = set() + for i in range(depth, 0, -1): + for declaration in declarations: + if not isinstance(declaration, PlaceholderType) and \ + (generics := [d for d in declaration.type if isinstance(d, GenericType)]): + for generic in generics: + if self.has_recursion_in_type(generic_types, generic.types): + generic_types = self.fix_recursion_in_type(generic_types, generic.types) + else: + generic_types.update(generic.types) + declarations = generic_types.copy() + generic_types.clear() + for declaration in declarations: + if not isinstance(declaration, PlaceholderType) and \ + (generics := [d for d in declaration.type if isinstance(d, GenericType)]): + for generic in generics: + if self.has_recursion_in_type(generic, types): + generic.types = self.fix_recursion_in_type(generic, types) + else: + generic.types.update(types) + + def visit_For(self, node: ast.For) -> Any: + self.visit_common_for(node) + + def visit_AsyncFor(self, node: ast.AsyncFor) -> Any: + self.visit_common_for(node) + + def visit_common_for(self, node: Union[ast.For, ast.AsyncFor]): + def handle_loop_variable(types, var): + declarations = self.get_declaration(var) + for declaration in declarations: + if declaration is None or isinstance(declaration, PlaceholderType): + assert isinstance(var, ast.Name) + if var.id == '_': + continue # TODO + r = create_range_from_ast_node(var) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(var.id, r.start_position.line) + new_variable = VariableDeclaration(var.id, qualified_name, + FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + else: + tt = types + if self.has_recursion_in_type(declaration, tt): + tt = self.fix_recursion_in_type(declaration, tt) + declaration.type = self.type_deduction.get_current_type(tt) + if len(declarations) == 0: + self.handle_new_variable(var) + + def handle_loop_target(types, elem): + if isinstance(elem, (ast.Name, ast.Starred)): + handle_loop_variable(types, elem) + elif isinstance(elem, (ast.Tuple, ast.List)): + for var in elem.elts: + handle_loop_target(types, var) + else: + assert False + + def action(): + types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(node.iter)) + if len(types) == 0: + self.handle_new_variable(node.target) + + for t in types: + if isinstance(t, GenericType): + handle_loop_target(t.types, node.target) + elif not isinstance(t, PlaceholderType): + iterable = self.get_iterable_type({t}) + if iterable is not None: + next_obj = self.get_next_type(iterable) + if next_obj is not None: + handle_loop_target(next_obj, node.target) + else: + self.handle_new_variable(node.target) + else: + self.handle_new_variable(node.target) + else: + self.handle_new_variable(node.target) + self.generic_visit(node) + + self.scope_manager.with_scope(LoopScope(), action) + + def handle_new_variable(self, node): + if isinstance(node, (ast.Name, ast.Starred)): + r = create_range_from_ast_node(node) + if isinstance(node, ast.Name): + id = node.id + elif isinstance(node, ast.Starred): + id = node.value.id + else: + assert False + if id == '_': + return # TODO + file_position = FilePosition(self.current_file, r) + declaration = self.scope_manager.get_declaration(id) + if not isinstance(declaration, PlaceholderType) and declaration.file_position == file_position: + return + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(id, r.start_position.line) + new_variable = VariableDeclaration(id, qualified_name, file_position) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(node, (ast.Tuple, ast.List)): + for var in node.elts: + self.handle_new_variable(var) + elif isinstance(node, (ast.Attribute, ast.Subscript)): + pass # TODO: handle subscript as loop variable + else: + assert False, f"Unhandled type: {type(node)}" + + def get_iterable_type(self, t: Set) -> Optional[Set]: + current_type = self.type_deduction.get_current_type(t) + for ct in current_type: + if isinstance(ct, ClassDeclaration): + for m in reversed(ct.methods): + if m.name == '__iter__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + for m in reversed(ct.static_methods): + if m.name == '__iter__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + return None + + def get_next_type(self, t: Set) -> Optional[Set]: + current_type = self.type_deduction.get_current_type(t) + for ct in current_type: + if isinstance(ct, ClassDeclaration): + for m in reversed(ct.methods): + if m.name == '__next__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + for m in reversed(ct.static_methods): + if m.name == '__next__': + ret = {tt for tt in self.type_deduction.get_current_type({m}) if + not isinstance(tt, PlaceholderType)} + if len(ret) > 0: + return ret + return None + + def visit_With(self, node: ast.With) -> Any: + self.common_with_visit(node) + + def visit_AsyncWith(self, node: ast.AsyncWith) -> Any: + self.common_with_visit(node) + + def common_with_visit(self, node: Union[ast.With, ast.AsyncWith]) -> Any: + def handle_vars(variables, types): + if isinstance(variables, ast.Name): + if variables.id == '_': + return # TODO + r = create_range_from_ast_node(variables) + qn = self.scope_manager.get_qualified_name_from_current_scope(variables.id, r.start_position.line) + new_variable = VariableDeclaration(variables.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(variables, (ast.List, ast.Tuple)): + for elem in variables.elts: + handle_vars(elem, types) + elif isinstance(variables, (ast.Attribute, ast.Subscript)): + pass # TODO: handle ast.Attribute and ast.Subscript eg: Lib > test > test_buffer.py + else: + assert False + + def action(): + for with_item in node.items: + if with_item.optional_vars is not None: + types = self.type_deduction.get_current_type( + self.type_deduction.deduct_type(with_item.context_expr)) + handle_vars(with_item.optional_vars, types) + self.generic_visit(node) + + self.scope_manager.with_scope(LoopScope(), action) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> Any: + def action(): + if node.name is not None and node.name != '_': # TODO + r = create_range_from_ast_node(node) + types = self.type_deduction.deduct_type(node.type) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(node.name, r.start_position.line) + new_variable = VariableDeclaration(node.name, qualified_name, FilePosition(self.current_file, r), types) + self.scope_manager.append_exception_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(ExceptionScope(), action) + + def visit_Global(self, node: ast.Global) -> Any: + r = create_range_from_ast_node(node) + for name in node.names: + reference = self.scope_manager.get_global_variable(name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + if reference is None: + reference = self.scope_manager.get_global_variable(name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + if reference is None: + v = VariableDeclaration(name, qualified_name, FilePosition.get_empty_file_position(), + {VariablePlaceholderType(name)}) + # TODO: global variable not found, eg: Lib > test > test_dummy_threading.py > global running + self.scope_manager.append_variable_to_current_scope( + GlobalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), v)) + continue + self.scope_manager.append_variable_to_current_scope( + GlobalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) + if reference is not None: + reference.usages.append(Usage(name, FilePosition(self.current_file, r))) + + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_ListComp(self, node: ast.ListComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_SetComp(self, node: ast.SetComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_DictComp(self, node: ast.DictComp) -> Any: + self.scope_manager.with_scope(GeneratorScope(), lambda: self.generic_visit(node)) + + def visit_Lambda(self, node: ast.Lambda) -> Any: + def action(): + for var in node.args.args: + if var.arg == '_': + continue # TODO + r = create_range_from_ast_node(var) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(var.arg, r.start_position.line) + new_variable = VariableDeclaration(var.arg, qualified_name, FilePosition(self.current_file, r)) + self.scope_manager.append_variable_to_current_scope(new_variable) + self.generic_visit(node) + + self.scope_manager.with_scope(LambdaScope(), action) + + def visit_comprehension(self, node: ast.comprehension) -> Any: + def handle_comprehension_target(target): + if isinstance(target, ast.Name): + if target.id == '_': + return + r = create_range_from_ast_node(target) + qn = self.scope_manager.get_qualified_name_from_current_scope(target.id, r.start_position.line) + types = self.type_deduction.get_current_type(self.type_deduction.deduct_type(node.iter)) + new_variable = VariableDeclaration(target.id, qn, FilePosition(self.current_file, r), types) + self.scope_manager.append_variable_to_current_scope(new_variable) + elif isinstance(target, ast.Tuple): + for var in target.elts: + handle_comprehension_target(var) + elif isinstance(target, ast.Subscript): + pass # TODO: handle subscript as comprehension target + else: + assert False, 'Comprehension target type not handled: ' + type(target) + + handle_comprehension_target(node.target) + self.generic_visit(node) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> Any: + r = create_range_from_ast_node(node) + for name in node.names: + reference = self.scope_manager.get_nonlocal_variable(name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(name, r.start_position.line) + if reference is None: + v = VariableDeclaration(name, qualified_name, FilePosition.get_empty_file_position(), + {VariablePlaceholderType(name)}) + # TODO: nonlocal variable declared after function declaration + self.scope_manager.append_variable_to_current_scope( + NonlocalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), v)) + continue + self.scope_manager.append_variable_to_current_scope( + NonlocalVariableDeclaration(name, qualified_name, FilePosition(self.current_file, r), reference)) + if reference is not None: + reference.usages.append(Usage(name, FilePosition(self.current_file, r))) + + def visit_Name(self, node: ast.Name) -> Any: + if isinstance(node.ctx, ast.Store): + var = self.get_declaration(node) + if var is not None and not isinstance(list(var)[0], PlaceholderType): + other = self.create_var_usage(node.id, node) + if not list(var)[0].is_same_usage(other): + self.append_variable_usage(node.id, node) + elif node.id == '_': + pass + else: + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager. \ + get_qualified_name_from_current_scope(node.id, r.start_position.line) + self.scope_manager.append_variable_to_current_scope( + VariableDeclaration(node.id, qualified_name, FilePosition(self.current_file, r))) + assert False, 'Assignment target, was not handled at ' + str(r) + elif isinstance(node.ctx, (ast.Load, ast.Del)): + self.append_variable_usage(node.id, node) + else: + assert False, "Unknown context" + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> Any: + self.append_variable_usage(node.attr, node) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> Any: + self.visit(node.func) + + for arg in node.args: + self.visit(arg) + for keyword in node.keywords: + self.visit(keyword.value) + + self.append_function_usage(node) + + def visit_Delete(self, node: ast.Delete) -> Any: + # TODO: remove variable (if not -> only runtime error) + self.generic_visit(node) + + def visit_If(self, node: ast.If) -> Any: + # TODO: can be recursive (eg. if..elif..elif..else..), is it a problem? + self.scope_manager.with_scope(ConditionalScope(), lambda: self.generic_visit(node)) + + def append_variable_usage(self, name: str, node: (ast.Name, ast.Attribute, ast.Subscript)) -> None: + usage = self.create_var_usage(name, node) + variable_declarations = self.get_declaration(node) + for var in variable_declarations: + if var is not None and not isinstance(var, PlaceholderType): + if isinstance(var, ImportedDeclaration): + var.imported_declaration.usages.append(usage) + else: + var.usages.append(usage) + logger.debug('Var Usage: ' + str(usage) + ' ' + var.get_type_repr()) + return + # TODO: annotations -> assert + assert True, "Variable not found: " + usage.name + " (usage: line - " + str(usage.file_position) + ")" + + def append_function_usage(self, node: ast.Call) -> None: + usage = self.create_function_usage(node) + mac = self.member_access_collector_type(node) + is_method_call = (len(mac.call_list) == 2 and + isinstance(mac.call_list[1], MemberAccessCollector.AttributeData) and + mac.call_list[1].name == 'self') + if not (len(mac.call_list) == 1 or is_method_call): + # functions from another module and modules + # return # TODO: handle these cases (type deduction) + pass + function_declarations = self.get_declaration(node) + if function_declarations is not None: + for func in function_declarations: + if func is not None and not isinstance(func, PlaceholderType): + if isinstance(func, ImportedDeclaration): + func.imported_declaration.usages.append(usage) + else: + func.usages.append(usage) + logger.debug('Func Usage: ' + str(usage) + ' ' + func.get_type_repr()) + return + if self.is_builtin(usage.name): + self.append_builtin_usage(usage) + return + # TODO: imported functions/methods from libraries not in the project (builtins!) + assert True, "Function not found: " + usage.name + " (usage: start position - " + str(usage.file_position) + ")" + + def is_builtin(self, name: str) -> bool: + return get_built_in_function(name) is not None + + def append_builtin_usage(self, usage: Usage): + bif = get_built_in_function(usage.name) + if bif is not None: + bif.usages.append(usage) + + def get_declaration(self, node: (ast.Name, ast.Attribute, ast.Call, ast.Subscript)) -> Optional[Set[Declaration]]: + if isinstance(node, ast.Name): + declaration = self.scope_manager.get_declaration(node.id) + if declaration is None: + return None + else: + return {declaration} + + mac = MemberAccessCollector(node) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and len(mac.call_list) == 1: + declaration = self.scope_manager.get_declaration(node.func.id) + if declaration is None or isinstance(declaration, PlaceholderType): + bif = get_built_in_function(node.func.id) + if bif is None: + return None + else: + return {bif} + else: + return {declaration} + elif isinstance(node, ast.Call) and len(mac.call_list) == 1 and not isinstance(node.func, ast.Subscript): + pass # print("NEED FIX: () operator called on lambda, operator or await") + + mac = MemberAccessCollector(node) + last = mac.call_list.pop(0) + types = self.type_deduction.get_member_access_type(mac) + declarations = set() + + for d in types: + if isinstance(d, ModuleVariableDeclaration): + if isinstance(d.imported_module, GlobalScope): + declaration = self.scope_manager.get_declaration_from_scope(last.name, d.imported_module) + if declaration is not None: + declarations.add(declaration) + elif isinstance(d.imported_module, PreprocessedFile): + pass + elif d.imported_module is None: + pass + else: + assert False + elif isinstance(d, (VariableDeclaration, FunctionDeclaration, ClassDeclaration)): + if isinstance(d, (TypeVariableDeclaration, FunctionVariableDeclaration)): + current_types = self.type_deduction.get_current_type({d.reference}) + else: + current_types = self.type_deduction.get_current_type(d.type) + for t in current_types: + if isinstance(t, ClassDeclaration): + if t is self.scope_manager.get_current_class_declaration(): + declaration = self.scope_manager.get_declaration_from_current_class(last.name) + if isinstance(declaration, PlaceholderType): + for cl in reversed(self.preprocessed_file.class_collector.classes): + if not cl.file_position == t.file_position: + continue + for a in reversed(cl.attributes): + if a.name == last.name: + declarations.add(a) + for m in reversed(cl.methods): + if m.name == last.name: + if isinstance(last, MemberAccessCollector.AttributeData): + declarations.add(MethodVariableDeclaration(m, t)) + else: + declarations.add(m) + else: + if isinstance(declaration, FunctionDeclaration) and \ + isinstance(last, MemberAccessCollector.AttributeData): + declarations.add(MethodVariableDeclaration(declaration, t)) + else: + declarations.add(declaration) + elif isinstance(last, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for a in t.attributes: + if a.name == last.name: + declarations.add(a) + for a in t.static_attributes: + if a.name == last.name: + declarations.add(a) + elif isinstance(last, MemberAccessCollector.MethodData): + for m in t.methods: + if m.name == last.name: + declarations.add(m) + for m in t.static_methods: + if m.name == last.name: + declarations.add(m) + else: + pass # print("NEED FIX: BuiltIn, InitVariablePlaceholderType, VariablePlaceholderType") + elif isinstance(d, (PlaceholderType, BuiltIn)): + pass + else: + assert False + + return declarations + + # TODO: change PlaceholderType-s, to actual type + # TODO: declaration is None -> what if not declared in global scope (eg. class member) - pass preprocessed 'scope'? + def post_process(self) -> None: + def post_process_variable(var: PreprocessedVariable, class_declaration: Optional[ClassDeclaration]) -> None: + var_declaration = None + if class_declaration is None: + var_declaration = self.scope_manager.get_global_scope().get_variable_by_name(var.name) + else: + for a in reversed(class_declaration.attributes): + if a.name == var.name: + var_declaration = a + break + for a in reversed(class_declaration.static_attributes): + if a.name == var.name: + var_declaration = a + break + if var_declaration is None: + return + var_declaration.usages.extend(var.usages) + for type_usage in var.type_usages: + type_usage.type.update(var_declaration.type) + + def post_process_function(func: PreprocessedFunction, class_declaration: Optional[ClassDeclaration]) -> None: + func_declaration = None + if class_declaration is None: + func_declaration = self.scope_manager.get_global_scope().get_function_by_name(func.name) + else: + for m in reversed(class_declaration.methods): + if m.name == func.name: + func_declaration = m + break + for m in reversed(class_declaration.static_methods): + if m.name == func.name: + func_declaration = m + break + if func_declaration is None: + return + func_declaration.usages.extend(func.usages) + for type_usage in func.type_usages: + type_usage.type.update(func_declaration.type) + + def post_process_class(pcl: PreprocessedClass, class_declaration: Optional[ClassDeclaration]) -> None: + current_class_declaration = None + if class_declaration is None: + current_class_declaration = self.scope_manager.get_global_scope().get_class_by_name(pcl.name) + else: + for cl in reversed(class_declaration.classes): + if c.name == cl.name: + current_class_declaration = c + break + + if current_class_declaration is None: + return + + for cv in pcl.attributes: + post_process_variable(cv, current_class_declaration) + for cf in pcl.methods: + post_process_function(cf, current_class_declaration) + for cc in pcl.classes: + post_process_class(cc, current_class_declaration) + + for type_usage in pcl.type_usages: + type_usage.type.add(current_class_declaration) + + for v in self.preprocessed_file.preprocessed_variables: + post_process_variable(v, None) + for f in self.preprocessed_file.preprocessed_functions: + post_process_function(f, None) + for c in self.preprocessed_file.class_collector.classes: + post_process_class(c, None) + + def create_var_usage(self, name: str, param: ast.expr) -> Usage: + r = create_range_from_ast_node(param) + return Usage(name, FilePosition(self.current_file, r)) + + def create_function_usage(self, func: ast.Call) -> Usage: + name = '' + h = ['func'] + while True: + if eval('hasattr(' + '.'.join(h) + ', "value")'): + if eval('isinstance(' + ".".join(h) + ', str)'): + break + else: + h.append('value') + elif eval('hasattr(' + '.'.join(h) + ', "func")'): + h.append('func') + else: + break + + n = [] + for i in range(len(h), 1, -1): + if eval('hasattr(' + '.'.join(h[:i]) + ', "id")'): + id_attr = "id" + elif eval('hasattr(' + '.'.join(h[:i]) + ', "attr")'): + id_attr = "attr" + else: + continue + n.append(eval("getattr(" + '.'.join(h[:i]) + ", '" + id_attr + "')")) + + # asdf = '.'.join(n) + + mac = MemberAccessCollector(func) + + if hasattr(func.func, 'id'): + name = func.func.id + elif hasattr(func.func, 'attr'): + name = func.func.attr + else: + # print("NEED FIX: usage when () operator called on return value") + assert True # call on return value (eg. f()(), g[i]()) + + r = create_range_from_ast_node(func) + return Usage(name, FilePosition(self.current_file, r)) + + @staticmethod + def is_static_method(node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> bool: + for decorator in node.decorator_list: + if isinstance(decorator, ast.Name) and decorator.id == 'staticmethod': + return True + return False + + def is_static_variable(self, node: ast.AST) -> bool: + return isinstance(self.scope_manager.get_current_scope(), ClassScope) and isinstance(node, ast.Name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py new file mode 100644 index 000000000..8e816e2ee --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector_interface.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod +from typing import Dict + +from cc_python_parser.base_data import ImportedDeclaration +from cc_python_parser.import_finder import ImportFinder +from cc_python_parser.member_access_collector import MemberAccessCollector +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.scope_manager import ScopeManager + + +class ISymbolCollector(ABC): + @abstractmethod + def collect_symbols(self): + pass + + +class SymbolCollectorBase(ISymbolCollector, ABC): + def __init__(self, preprocessed_file: PreprocessedFile, import_finder: ImportFinder): + self.preprocessed_file: PreprocessedFile = preprocessed_file + self.import_finder: ImportFinder = import_finder + self.member_access_collector_type = MemberAccessCollector + self.imported_declaration_scope_map: Dict[ImportedDeclaration, ScopeManager] = {} + + +class IFunctionSymbolCollector(ISymbolCollector, ABC): + def __init__(self): + self.function = None diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py new file mode 100644 index 000000000..c140bf10c --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_finder.py @@ -0,0 +1,10 @@ +from typing import Optional +from abc import ABC, abstractmethod + +from cc_python_parser.base_data import Declaration + + +class SymbolFinder(ABC): + @abstractmethod + def get_declaration(self, name: str) -> Optional[Declaration]: + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_data.py b/plugins/python/parser/src/scripts/cc_python_parser/type_data.py new file mode 100644 index 000000000..7e2956d26 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_data.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod + + +class DeclarationType(ABC): + def __init__(self, name: str, qualified_name: str): + self.name = name + self.qualified_name = qualified_name + + @abstractmethod + def get_type_repr(self) -> str: + pass + + @abstractmethod + def __hash__(self): + pass + + @abstractmethod + def __eq__(self, other): + pass + + @abstractmethod + def __ne__(self, other): + pass + + +class PlaceholderType(DeclarationType): + def __init__(self, name: str): + DeclarationType.__init__(self, name, "") + + def get_type_repr(self) -> str: + return 'Placeholder' + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return isinstance(other, type(self)) and self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + +class VariablePlaceholderType(PlaceholderType): + pass + + +class InitVariablePlaceholderType(VariablePlaceholderType): + pass + + +class FunctionPlaceholderType(PlaceholderType): + pass diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py b/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py new file mode 100644 index 000000000..fdb029e85 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py @@ -0,0 +1,610 @@ +import ast +import typing +from functools import singledispatchmethod +from typing import Optional, Any, Union + +from cc_python_parser import built_in_types +from cc_python_parser.base_data import Declaration, ImportedDeclaration, Usage +from cc_python_parser.built_in_functions import get_built_in_function +from cc_python_parser.built_in_operators import get_built_in_operator +from cc_python_parser.built_in_types import Boolean, Dictionary, Set, Tuple, String, Integer, Float, Bytes, \ + EllipsisType, NoneType, Complex, RangeType, BuiltIn, Type, GenericBuiltInType, NotImplementedType, \ + get_built_in_type, GenericType, Generator +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.common.hashable_list import OrderedHashableList +from cc_python_parser.common.utils import create_range_from_ast_node +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory +from cc_python_parser.member_access_collector import MemberAccessCollector, MemberAccessCollectorIterator +from cc_python_parser.preprocessed_data import PreprocessedDeclaration +from cc_python_parser.preprocessed_file import PreprocessedFile +from cc_python_parser.scope_manager import ScopeManager +from cc_python_parser.symbol_collector_interface import SymbolCollectorBase +from cc_python_parser.type_data import DeclarationType, PlaceholderType, VariablePlaceholderType, FunctionPlaceholderType, \ + InitVariablePlaceholderType +from cc_python_parser.type_hint_data import TypeHintType +from cc_python_parser.variable_data import VariableDeclaration, FunctionVariableDeclaration, TypeVariableDeclaration, \ + MethodVariableDeclaration + + +class TypeDeduction: + def __init__(self, symbol_collector: SymbolCollectorBase, scope_manager: ScopeManager, + preprocessed_file: PreprocessedFile, + function_symbol_collector_factory: FunctionSymbolCollectorFactory): + self.scope_manager: ScopeManager = scope_manager + self.preprocessed_file: PreprocessedFile = preprocessed_file + self.symbol_collector: SymbolCollectorBase = symbol_collector + self.function_symbol_collector_factory: FunctionSymbolCollectorFactory = function_symbol_collector_factory + self.container_recursion_counter: int = 0 + self.container_max_recursion: int = 5 + + def deduct_type(self, node: ast.AST) -> typing.Set: + types = self.get_type(node) + if not isinstance(types, typing.Set): + return {types} + return types + + def get_current_type(self, types: typing.Set[DeclarationType]) \ + -> typing.Set[Union[ + ClassDeclaration, BuiltIn, FunctionVariableDeclaration, TypeVariableDeclaration, PlaceholderType]]: + def get_current_type_impl(declaration_type: DeclarationType) \ + -> typing.Set[Union[ + ClassDeclaration, BuiltIn, FunctionVariableDeclaration, TypeVariableDeclaration, PlaceholderType]]: + if isinstance(declaration_type, PlaceholderType): + if isinstance(declaration_type, InitVariablePlaceholderType): + return {declaration_type} + declaration = self.scope_manager.get_declaration(declaration_type.name) + if isinstance(declaration, PlaceholderType) and declaration_type.name == declaration.name: + return {declaration} + if declaration in types: + return set() + tt = {d for d in declaration.type if d not in types} + return self.get_current_type(tt) + elif isinstance(declaration_type, FunctionDeclaration): + return self.get_current_type(declaration_type.type) + elif isinstance(declaration_type, + (ClassDeclaration, BuiltIn, FunctionVariableDeclaration, TypeVariableDeclaration)): + return {declaration_type} + elif isinstance(declaration_type, VariableDeclaration): + return self.get_current_type(declaration_type.type) + elif isinstance(declaration_type, TypeHintType): + return {declaration_type.hinted_type} + elif isinstance(declaration_type, PreprocessedDeclaration): + return {declaration_type} + else: + assert False + + current_types = set() + for t in types: + if t is None: + continue + current_types.update(get_current_type_impl(t)) + return current_types + + @singledispatchmethod + def get_type(self, node: ast.AST) -> Any: + assert False, "Unknown type: " + str(type(node)) + + @get_type.register + def _(self, node: ast.BoolOp): + return Boolean() + + @get_type.register + def _(self, node: ast.NamedExpr): # (x := y) eg. if (y := f(x)) is not None: ... + pass + + @get_type.register + def _(self, node: ast.BinOp): + left_operand_type = self.get_current_type(self.deduct_type(node.left)) + bio = get_built_in_operator(node.op) + assert bio is not None + types = set() + for t in left_operand_type: + if isinstance(t, (BuiltIn, PlaceholderType)): + types.add(t) # TODO: handle numbers + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: check this (eg. base class is BuiltIn) + types.add(t) + else: + pass # print("???") + return types # type(left_operand_type.op) + + @get_type.register + def _(self, node: ast.UnaryOp): + operand = self.get_current_type(self.deduct_type(node.operand)) + bio = get_built_in_operator(node.op) + assert bio is not None + types = set() + for t in operand: + if isinstance(t, BuiltIn): + types.add(t) + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: preprocessed method + types.add(t) + else: + pass # print("???") + return types + + @get_type.register + def _(self, node: ast.Lambda): + # add node.args to tmp_scope + return self.get_type(node.body) + + @get_type.register + def _(self, node: ast.IfExp): + types = self.deduct_type(node.body) + types.update(self.deduct_type(node.orelse)) + return types + + @get_type.register + def _(self, node: ast.Dict): + return Dictionary(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.Set): + return Set(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.ListComp): + return built_in_types.List() # generic types? + + @get_type.register + def _(self, node: ast.SetComp): + return Set() # generic types? + + @get_type.register + def _(self, node: ast.DictComp): + return Dictionary() # generic types? + + @get_type.register + def _(self, node: ast.GeneratorExp): + g = Generator() + for generator in node.generators: + g.types.update(self.deduct_type(generator.iter)) + return g # tmp_var := type(iter) -> type(target) + + @get_type.register + def _(self, node: ast.Await): + return self.get_type(node.value) + + @get_type.register + def _(self, node: ast.Compare): + # TODO: handle PlaceHolderType + left_values = self.deduct_type(node.left) + types = set() + assert len(node.comparators) == len(node.ops) + for i in range(0, len(node.comparators)): + bio = get_built_in_operator(node.ops[i]) + assert bio is not None + left_operand_type = self.get_current_type(left_values) + for t in left_operand_type: + if isinstance(t, BuiltIn): + types.add(Boolean()) + elif isinstance(t, ClassDeclaration): + override = [x for x in t.methods if x.name in bio.get_override()] + if len(override) > 0: + types.add(override[0]) + else: + # TODO: check this (eg. base class is BuiltIn) + types.add(t) + else: + pass # print("???") + left_values = types + return left_values + + @get_type.register + def _(self, node: ast.Call): + if hasattr(node.func, 'id'): + # TODO: check if builtin class ctor-s are not hidden by other (custom) functions/classes + # TODO: no args check (builtin is not hidden + correct code -> should not be a problem) + built_in_function = get_built_in_function(node.func.id) + if built_in_function is not None: + return built_in_function + elif node.func.id == 'TypeVar': + return Type() + return self.get_member_access_type(MemberAccessCollector(node), + FilePosition(self.scope_manager.current_file, + create_range_from_ast_node(node))) + + @get_type.register + def _(self, node: ast.FormattedValue): # {x} in a joined string + return self.get_type(node.value) # cannot be on right side without f-string? + + @get_type.register + def _(self, node: ast.JoinedStr): # f"... {x} ... {y} ..." + return String() + + @get_type.register + def _(self, node: ast.Constant): + if isinstance(node.value, int): + if isinstance(node.value, bool): + return Boolean() + else: + return Integer() + elif isinstance(node.value, float): + return Float() + elif isinstance(node.value, bytes): + return Bytes() + elif isinstance(node.value, complex): + return Complex() + elif isinstance(node.value, str): + return String() + elif node.value is None: + return NoneType() + elif node.value is Ellipsis: + return EllipsisType() + return set() + + @get_type.register + def _(self, node: ast.Attribute): + return self.get_member_access_type(MemberAccessCollector(node), + FilePosition(self.scope_manager.current_file, + create_range_from_ast_node(node))) + + @get_type.register + def _(self, node: ast.Subscript): + if isinstance(node.slice, ast.Index): + if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) + return Integer() + else: + if isinstance(node.value, (ast.Name, ast.Attribute, ast.Subscript, ast.Call, ast.Tuple, + ast.ListComp, ast.List)): + t = self.deduct_type(node.value) + elif isinstance(node.value, (ast.Dict, ast.DictComp)): + t = set() + elif isinstance(node.value, ast.Constant): + t = {String()} # TODO: can be other than str? + elif isinstance(node.value, (ast.BinOp, ast.UnaryOp)): + t = set() + operand_types = self.deduct_type(node.value) + bio = get_built_in_operator(node.value.op) + assert bio is not None + for operand_type in self.get_current_type(operand_types): + if isinstance(operand_type, ClassDeclaration): + override = [x for x in operand_type.methods if x.name in bio.get_override()] + if len(override) > 0: + t.add(override[0]) + else: + assert False, f"Unhandled type: {type(node.value)}" + generic_types = set() + subscript_type_sets = [a.type for a in t if not isinstance(a, PlaceholderType)] + for subscript_types in subscript_type_sets: + for subscript_type in subscript_types: + if isinstance(subscript_type, GenericType): + for generic_type in subscript_type.types: + if isinstance(generic_type, ImportedDeclaration): + generic_types.add(generic_type.imported_declaration) + else: + generic_types.add(generic_type) + return generic_types + elif isinstance(node.slice, (ast.Slice, ast.ExtSlice)): # ExtSlice -> array type (module arr) + if isinstance(self.get_type(node.value), RangeType): # TODO: get_type -> set(..) + return RangeType() + else: + return set() # type(node.value)[type(node.value->GenericType)] + return set() # {type(value)...} + + @get_type.register + def _(self, node: ast.Starred): # unpack: * (eg. list, tuple) and ** (eg. dictionary) + return set() # could be same as iterator + + @get_type.register + def _(self, node: ast.Name): + bit = get_built_in_type(node.id) + if bit is not None: + return bit + if node.id == 'NotImplemented': + return NotImplementedType() + elif node.id == 'Ellipsis': + return EllipsisType() + return self.scope_manager.get_variable_declaration(node.id) + + @get_type.register + def _(self, node: ast.List): + return built_in_types.List(self.get_element_types(node)) + + @get_type.register + def _(self, node: ast.Tuple): + return Tuple(self.get_element_types(node)) + + @get_type.register(ast.Return) + @get_type.register(ast.Yield) + @get_type.register(ast.YieldFrom) + def _(self, node): + return self.get_type_of_function(node) + + @get_type.register + def _(self, node: ast.Expr): + return self.get_type(node.value) + + def get_element_types(self, node: ast.AST): + # TODO: need better soultion eg: Lib > test > test_parser.py (tuple) + if self.container_recursion_counter < self.container_recursion_counter: + return set() + self.container_recursion_counter += 1 + element_types = set() + if isinstance(node, ast.Dict): + elements = getattr(node, 'values') + else: + assert hasattr(node, 'elts') + elements = getattr(node, 'elts') + for elem in elements: + elem_type = self.get_type(elem) + if elem_type is not None: + if isinstance(elem_type, typing.Set): + element_types.update(elem_type) + else: + element_types.add(elem_type) + self.container_recursion_counter -= 1 + return element_types + + def get_member_access_type(self, mac: MemberAccessCollector, pos: Optional[FilePosition] = None): + if len(mac.call_list) == 0: + return set() + iterator = MemberAccessCollectorIterator(mac) + declaration = self.scope_manager.get_declaration_from_member_access(iterator) + if (declaration is None or isinstance(declaration, PlaceholderType)) and pos is not None: + if iterator.is_iteration_started(): + a = iterator.get_current() + else: + a = iterator.get_first() + + # TODO: is empty FilePosition correct? + if isinstance(a, MemberAccessCollector.MethodData): + b = [x for x in self.preprocessed_file.preprocessed_functions if x.name == a.name] + c = [x for x in self.preprocessed_file.class_collector.classes if x.name == a.name] + for i in b: + i.usages.append(Usage(a.name, pos)) + for i in c: + i.usages.append(Usage(a.name, pos)) + elif isinstance(a, (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + b = [x for x in self.preprocessed_file.preprocessed_variables if x.name == a.name] + for i in b: + i.usages.append(Usage(a.name, pos)) + declarations = {declaration} + if iterator.is_iteration_over() or not iterator.is_iteration_started(): + if not any(isinstance(x, PlaceholderType) for x in declarations) and \ + any(any(isinstance(x, PlaceholderType) for x in y.type) for y in declarations): + if any(isinstance(x, FunctionVariableDeclaration) for x in declarations): + pass # print() + else: + declarations = self.get_current_type_of_placeholder_function(declarations, mac.call_list[-1]) + return declarations + prev_member = iterator.get_current() + next(iterator) + for member in mac.call_list[iterator.index::-1]: + if len(declarations) == 0 or all(x is None for x in declarations) or \ + all(isinstance(x, PlaceholderType) for x in declarations): + return set() + declarations = self.get_member_declarations(declarations, prev_member, member) + prev_member = member + return declarations + + def get_member_declarations(self, declarations: typing.Set[DeclarationType], + current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + member_declarations = set() + for declaration in declarations: + for declaration_type in self.get_current_type(declaration.type): + if isinstance(declaration_type, PlaceholderType) and isinstance(declaration, FunctionDeclaration): + d = self.get_current_type_of_placeholder_function({declaration}, current_member) + for dt in d: + for declaration_current_type in self.get_possible_types(dt): + if isinstance(declaration_current_type, BuiltIn): + continue + elif isinstance(declaration_current_type, ClassDeclaration): + member_declarations.update( + self.get_member(declaration_current_type, current_member, next_member)) + elif isinstance(declaration_current_type, (PlaceholderType, VariablePlaceholderType)): + continue # TODO: handle this (eg. function parameters) + else: + assert False + # member_declarations.update(d) + continue # TODO: try to get current value of function with the arguments from 'mac' + for declaration_current_type in self.get_possible_types(declaration_type): + if isinstance(declaration_current_type, BuiltIn): + self.get_member_of_builtin(declaration_current_type, current_member, next_member) + elif isinstance(declaration_current_type, ClassDeclaration): + member_declarations.update(self.get_member( + declaration_current_type, current_member, next_member)) + elif isinstance(declaration_current_type, VariablePlaceholderType): + continue # TODO: handle this (eg. function parameters) + elif isinstance(declaration_current_type, InitVariablePlaceholderType): + continue + elif isinstance(declaration_current_type, PlaceholderType): + continue + else: + self.get_possible_types(declaration_type) + assert False, f"Unhandled type: {type(declaration_current_type)}" + return member_declarations + + def get_member(self, declaration: Declaration, current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + types = self.get_possible_types(declaration) + declarations = set() + pf = None + for t in types: + if isinstance(t, ClassDeclaration): + if t is self.scope_manager.get_current_class_declaration(): + pf = self.preprocessed_file + pc = None + for c in self.preprocessed_file.class_collector.classes: + if c.file_position == t.file_position: + if isinstance(next_member, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for attr in reversed(c.attributes): + if attr.name == next_member.name: + pass + elif isinstance(next_member, MemberAccessCollector.MethodData): + for method in reversed(c.methods): + if method.name == next_member.name: + arguments = self.collect_current_argument_types(next_member) + # t = self.evaluate_function_def(method.node, call_on, arguments) + break + if pc is not None: + pass + # else: + if isinstance(next_member, + (MemberAccessCollector.AttributeData, MemberAccessCollector.SubscriptData)): + for attr in reversed(t.attributes): + if attr.name == next_member.name: + declarations.add(attr) + for attr in reversed(t.static_attributes): + if attr.name == next_member.name: + declarations.add(attr) + elif isinstance(next_member, MemberAccessCollector.MethodData): + for method in reversed(t.methods): + if method.name == next_member.name: + declarations.add(method) + for method in reversed(t.static_methods): + if method.name == next_member.name: + declarations.add(method) + elif isinstance(next_member, MemberAccessCollector.SubscriptData): + # TODO: subscript elements type -> init/call method + pass + elif isinstance(next_member, MemberAccessCollector.LambdaData): + # TODO: member.node type -> init/call method + pass + elif isinstance(next_member, MemberAccessCollector.OperatorData): + # TODO: member.node type -> init/call method + pass + elif isinstance(t, BuiltIn): + continue + else: + assert False + return declarations + + def get_member_of_builtin(self, builtin: BuiltIn, current_member: MemberAccessCollector.MemberData, + next_member: MemberAccessCollector.MemberData) \ + -> typing.Set[Union[VariableDeclaration, FunctionDeclaration]]: + if isinstance(builtin, GenericBuiltInType) and isinstance(current_member, MemberAccessCollector.SubscriptData): + return builtin.types + return set() + + # evaluate declaration to get type + def get_possible_types(self, declaration: DeclarationType) -> typing.Set[Union[ClassDeclaration, BuiltIn]]: + types = set() + # TODO: handle BuiltIn-s (Module!) + if isinstance(declaration, BuiltIn): + return {declaration} + elif isinstance(declaration, Declaration): + for t in declaration.type: + if isinstance(t, (ClassDeclaration, BuiltIn, PlaceholderType)): + types.add(t) + elif isinstance(t, (VariableDeclaration, FunctionDeclaration)): + types.update(self.get_possible_types(t)) + else: + assert False, "Unknown type: " + str(type(t)) + elif isinstance(declaration, PlaceholderType): + # TODO: get current value of function or variable + return self.get_current_type({self.scope_manager.get_declaration(declaration.name)}) + else: + assert False, "Unknown declaration type: " + str(type(declaration)) + return types + + def get_current_type_of_placeholder_function(self, declarations: typing.Set, + member: MemberAccessCollector.MemberData) \ + -> typing.Set[DeclarationType]: + types = set() + + for declaration in declarations: + if isinstance(declaration, ImportedDeclaration): + if declaration not in self.symbol_collector.imported_declaration_scope_map: + continue # TODO: fix this, eg: Lib > unittest > test > test_discovery.py > in def restore_listdir + scope_manager = self.symbol_collector.imported_declaration_scope_map[declaration] + func_def = scope_manager.placeholder_function_declaration_cache. \ + get_function_def(declaration.imported_declaration) + elif isinstance(declaration, FunctionVariableDeclaration): + func_def = self.scope_manager.placeholder_function_declaration_cache. \ + get_function_def(declaration.reference) + else: + func_def = self.scope_manager.placeholder_function_declaration_cache.get_function_def(declaration) + # assert func_def is not None # VariableDeclaration? + if func_def is None: + continue + + arguments = OrderedHashableList() + if isinstance(declaration, MethodVariableDeclaration): + arguments.append((declaration.self, None)) + elif func_def.is_method: + arguments.append((self.get_current_type({declaration}), None)) + arguments.extend(self.collect_current_argument_types(member)) + declaration_type = self.scope_manager.placeholder_function_declaration_cache. \ + get_functions_return_type(declarations, arguments) + if declaration_type is not None and len(declaration_type) > 0: + types.update(declaration_type) + else: + types.update(self.evaluate_function_def(func_def.function, arguments)) + return types + + def collect_current_argument_types(self, method_data: MemberAccessCollector.MemberData) -> OrderedHashableList: + arguments = OrderedHashableList() + if isinstance(method_data, MemberAccessCollector.MethodData): + for arg in method_data.arguments: + arguments.append((self.deduct_type(arg), None)) + + for kw in method_data.keywords: + arguments.append((self.deduct_type(kw.argument), kw.name)) + return arguments + + def evaluate_function_def(self, function_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], + arguments: OrderedHashableList) -> typing.Set[DeclarationType]: + types = set() + if len(arguments) == 0: + sc = self.function_symbol_collector_factory.get_function_symbol_collector( + self.symbol_collector, function_def, []) + sc.collect_symbols() + types.update(sc.function.type) + else: + for args in self.get_argument_combinations(arguments): + sc = self.function_symbol_collector_factory.get_function_symbol_collector( + self.symbol_collector, function_def, args) + sc.collect_symbols() + types.update(sc.function.type) + return types + + def get_argument_combinations(self, arguments: + OrderedHashableList[typing.Tuple[typing.Set[DeclarationType], Optional[str]]]) \ + -> typing.Iterator[OrderedHashableList[typing.Tuple[DeclarationType, Optional[str]]]]: + if len(arguments) == 0: + return OrderedHashableList() + for argument in arguments[0][0]: + head_argument = OrderedHashableList([(argument, arguments[0][1])]) + if len(arguments) == 1: + yield head_argument + else: + for tail_arguments in self.get_argument_combinations(arguments[1::]): + yield head_argument + tail_arguments + + def get_type_of_function(self, node: Union[ast.Return, ast.Yield, ast.YieldFrom]): + if node.value is None: + return set() + + types = self.get_type(node.value) + if types is None: + return set() + fixed_types = set() + if isinstance(types, typing.Set): + fixed_types.update(map(lambda x: self.fix_placeholder(x), types)) + else: + fixed_types.add(self.fix_placeholder(types)) + return fixed_types + + def fix_placeholder(self, declaration_type: DeclarationType): + if any(isinstance(x, PlaceholderType) for x in self.get_current_type({declaration_type})): + if isinstance(declaration_type, VariableDeclaration): + return VariablePlaceholderType(declaration_type.name) + elif isinstance(declaration_type, FunctionDeclaration): + return FunctionPlaceholderType(declaration_type.name) + return declaration_type diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py b/plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py new file mode 100644 index 000000000..f54e7cff6 --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_hint_data.py @@ -0,0 +1,23 @@ +from typing import Union + +from cc_python_parser.built_in_types import BuiltIn +from cc_python_parser.class_data import ClassDeclaration +from cc_python_parser.type_data import DeclarationType + + +class TypeHintType(DeclarationType): + def __init__(self, hinted_type: Union[ClassDeclaration, BuiltIn]): + super().__init__(hinted_type.name, hinted_type.qualified_name) + self.hinted_type = hinted_type + + def get_type_repr(self) -> str: + return 'Hinted type: ' + self.hinted_type.get_type_repr() + + def __hash__(self): + return hash(self.hinted_type) + + def __eq__(self, other): + return isinstance(other, TypeHintType) and self.hinted_type == other.hinted_type + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py b/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py new file mode 100644 index 000000000..ee9ad52bb --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py @@ -0,0 +1,101 @@ +from pathlib import PurePath +from typing import Optional, Set + +from cc_python_parser.built_in_types import Module, AnyType, Method, Function, Type +from cc_python_parser.common.file_position import FilePosition +from cc_python_parser.base_data import Declaration, TypeDeclaration, ImportedDeclaration, ReferenceDeclaration +from cc_python_parser.function_data import FunctionDeclaration +from cc_python_parser.persistence.base_dto import UsageDTO +from cc_python_parser.persistence.variable_dto import VariableDeclarationDTO +from cc_python_parser.type_data import PlaceholderType + + +class VariableDeclaration(Declaration): + # var_type: Optional[Set[Union[cd.ClassDeclaration, BuiltIn]]] - circular import + def __init__(self, name: str, qualified_name: str, pos: FilePosition, declaration_type: Optional[Set] = None): + super().__init__(name, qualified_name, pos, declaration_type) + # self.type: VariableDeclaration and FunctionDeclaration type can change + # (eg. new assignment with different type or redefinition) + + def create_dto(self) -> VariableDeclarationDTO: + usages = [] + for usage in self.usages: + usages.append(UsageDTO(usage.file_position)) + types = set() + for t in self.type: + if isinstance(t, PlaceholderType): + types.add(AnyType().qualified_name) + elif isinstance(t, MethodVariableDeclaration): + types.add(Method().qualified_name) + elif isinstance(t, FunctionVariableDeclaration): + types.add(Function().qualified_name) + elif isinstance(t, TypeVariableDeclaration): + types.add(Type().qualified_name) + elif isinstance(t, ImportedDeclaration): + types.add(t.imported_declaration.qualified_name) + elif isinstance(t, ReferenceDeclaration): + types.add(t.reference.qualified_name) + else: + types.add(t.qualified_name) + return VariableDeclarationDTO(self.name, self.qualified_name, self.file_position, types, usages) + + +class StaticVariableDeclaration(VariableDeclaration): + pass + + +class ReferenceVariableDeclaration(VariableDeclaration, ReferenceDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: Declaration): + VariableDeclaration.__init__(self, name, qualified_name, pos, reference.type) + ReferenceDeclaration.__init__(self, reference) + + +class NonlocalVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): + super().__init__(name, qualified_name, pos, reference) + + +class GlobalVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: VariableDeclaration): + super().__init__(name, qualified_name, pos, reference) + + +# TODO: short/long name? (in case of import) +class ModuleVariableDeclaration(VariableDeclaration): + # module: Union[GlobalScope, PreprocessedFile] - circular import + def __init__(self, name: str, location: PurePath, pos: FilePosition, module): + super().__init__(name, "", pos, {Module()}) # TODO: need qualified name? + self.imported_module = module + self.imported_module_location: PurePath = location + + +class ImportedVariableDeclaration(VariableDeclaration, ImportedDeclaration[VariableDeclaration]): + def __init__(self, name: str, pos: FilePosition, var_declaration: VariableDeclaration, module): + VariableDeclaration.__init__(self, name, "", pos, var_declaration.type) + ImportedDeclaration.__init__(self, var_declaration, module, pos) + + +class TypeVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, type_ref: TypeDeclaration): + super().__init__(type_ref.name, type_ref.qualified_name, type_ref.file_position, type_ref) + + def get_type_repr(self) -> str: + return '[TypeVariable(' + self.reference.name + ')]' + + +class FunctionVariableDeclaration(ReferenceVariableDeclaration): + def __init__(self, func_ref: FunctionDeclaration): + super().__init__(func_ref.name, func_ref.qualified_name, func_ref.file_position, func_ref) + + def get_type_repr(self) -> str: + return '[FunctionVariable(' + self.reference.name + ')]' + + +class MethodVariableDeclaration(FunctionVariableDeclaration): + # self_var: ClassDeclaration + def __init__(self, func_ref: FunctionDeclaration, self_var): + super().__init__(func_ref) + self.self = self_var + + def get_type_repr(self) -> str: + return '[MethodVariable(' + self.reference.name + ')]' From 14e3292972e0ee35cb61205b63aed86481c1c03d Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Thu, 14 Jan 2021 18:34:31 +0100 Subject: [PATCH 15/39] Add CodeBites option to the Python plugin. --- plugins/python/webgui/js/pythonMenu.js | 46 ++++++++++++++++++++ webgui/scripts/codecompass/view/codeBites.js | 7 ++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/plugins/python/webgui/js/pythonMenu.js b/plugins/python/webgui/js/pythonMenu.js index 7f50186db..a69a0248e 100644 --- a/plugins/python/webgui/js/pythonMenu.js +++ b/plugins/python/webgui/js/pythonMenu.js @@ -75,4 +75,50 @@ function (topic, Menu, MenuItem, PopupMenuItem, astHelper, model, viewHandler){ type : viewHandler.moduleType.TextContextMenu, service : model.pythonservice }); + + var diagrams = { + id : 'python-text-diagrams', + render : function (nodeInfo, fileInfo) { + if (!nodeInfo || !fileInfo) + return; + + var submenu = new Menu(); + + // var diagramTypes = model.pythonservice.getDiagramTypes(nodeInfo.id); + // for (diagramType in diagramTypes) + // submenu.addChild(new MenuItem({ + // label : diagramType, + // type : diagramType, + // onClick : function () { + // var that = this; + + // topic.publish('codecompass/openDiagram', { + // handler : 'python-ast-diagram', + // diagramType : diagramTypes[that.type], + // node : nodeInfo.id + // }); + // } + // })); + + submenu.addChild(new MenuItem({ + label : "CodeBites", + onClick : function () { + topic.publish('codecompass/codebites', { + node : nodeInfo + }); + } + })); + + //if (Object.keys(diagramTypes).length !== 0) + return new PopupMenuItem({ + label : 'Diagrams', + popup : submenu + }); + } + }; + + viewHandler.registerModule(diagrams, { + type : viewHandler.moduleType.TextContextMenu, + service : model.pythonservice + }); }); \ No newline at end of file diff --git a/webgui/scripts/codecompass/view/codeBites.js b/webgui/scripts/codecompass/view/codeBites.js index 7d2c8f4fe..3dcc8ff56 100644 --- a/webgui/scripts/codecompass/view/codeBites.js +++ b/webgui/scripts/codecompass/view/codeBites.js @@ -339,7 +339,12 @@ function (declare, array, dom, style, topic, on, ContentPane, ResizeHandle, fPos.file = this.astNodeInfo.range.file; var astNodeInfo = languageService.getAstNodeInfoByPosition(fPos); - var refTypes = model.cppservice.getReferenceTypes(astNodeInfo.id); + var service = model.getLanguageService(fileInfo.type); + if(service){ + var refTypes = service.getReferenceTypes(astNodeInfo.id); + } else { + return; + } var defAstNodeInfo = languageService.getReferences( astNodeInfo.id, refTypes["Definition"])[0]; From ea021a6da508bb3129b52bc7e87e27725a802d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Fri, 15 Jan 2021 15:11:14 +0100 Subject: [PATCH 16/39] Build Boost with python3 binary when creating tarballs. --- .gitlab/build-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/build-deps.sh b/.gitlab/build-deps.sh index 02398cf05..5c353708f 100644 --- a/.gitlab/build-deps.sh +++ b/.gitlab/build-deps.sh @@ -313,7 +313,7 @@ if [ ! -f $DEPS_INSTALL_RUNTIME_DIR/boost-install/lib/libboost_program_options.s ./bootstrap.sh \ --prefix=$DEPS_INSTALL_RUNTIME_DIR/boost-install \ - --with-python=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin/python + --with-python=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin/python3 ./b2 -j $(nproc) install rm -f $PACKAGES_DIR/boost_1_74_0.tar.gz From a011a5cadafdca84dc91dfde9c2649761215b491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Thu, 28 Jan 2021 15:19:52 +0100 Subject: [PATCH 17/39] Adding Python to CMAKE_PREFIX_PATH --- .gitlab/build-codecompass.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/build-codecompass.sh b/.gitlab/build-codecompass.sh index 5d080be18..09ed61d66 100644 --- a/.gitlab/build-codecompass.sh +++ b/.gitlab/build-codecompass.sh @@ -47,7 +47,8 @@ export CMAKE_PREFIX_PATH=$DEPS_INSTALL_RUNTIME_DIR/libgit2-install\ :$DEPS_INSTALL_RUNTIME_DIR/odb-install\ :$DEPS_INSTALL_RUNTIME_DIR/thrift-install\ :$DEPS_INSTALL_RUNTIME_DIR/boost-install\ -:$DEPS_INSTALL_RUNTIME_DIR/llvm-install +:$DEPS_INSTALL_RUNTIME_DIR/llvm-install\ +:$DEPS_INSTALL_RUNTIME_DIR/python-install export GTEST_ROOT=$DEPS_INSTALL_BUILD_DIR/gtest-install From 4a0faacdd7732aec1ef79f3b785bdadc342c31cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Thu, 28 Jan 2021 20:40:20 +0100 Subject: [PATCH 18/39] Build Python with dynamic library. --- .gitlab/build-codecompass.sh | 1 + .gitlab/build-deps.sh | 3 +++ .gitlab/cc-env.sh | 1 + 3 files changed, 5 insertions(+) diff --git a/.gitlab/build-codecompass.sh b/.gitlab/build-codecompass.sh index 09ed61d66..89b52509e 100644 --- a/.gitlab/build-codecompass.sh +++ b/.gitlab/build-codecompass.sh @@ -41,6 +41,7 @@ export LDFLAGS="-Wl,-rpath-link,$DEPS_INSTALL_RUNTIME_DIR/openssl-install/lib "\ "-Wl,-rpath-link,$DEPS_INSTALL_RUNTIME_DIR/postgresql-install/lib" export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/odb-install/lib\ +:$DEPS_INSTALL_RUNTIME_DIR/python-install/lib\ :$LD_LIBRARY_PATH export CMAKE_PREFIX_PATH=$DEPS_INSTALL_RUNTIME_DIR/libgit2-install\ diff --git a/.gitlab/build-deps.sh b/.gitlab/build-deps.sh index 5c353708f..74c2b8660 100644 --- a/.gitlab/build-deps.sh +++ b/.gitlab/build-deps.sh @@ -240,11 +240,14 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/openssl-install/lib:$LD_LIBRARY --quiet \ --prefix=$DEPS_INSTALL_RUNTIME_DIR/python-install \ --with-openssl=$DEPS_INSTALL_RUNTIME_DIR/openssl-install \ + --enable-shared \ --enable-optimizations make install --quiet --jobs $(nproc) rm -f $PACKAGES_DIR/Python-3.9.0.tar.xz export PATH=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin:$PATH +export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/python-install/lib:$LD_LIBRARY_PATH +# --enabled-shared: build libpython.so instead of libpython.a, must be on LD_LIBRARY_PATH ############## # LLVM/Clang # diff --git a/.gitlab/cc-env.sh b/.gitlab/cc-env.sh index f018b8177..ddf0e5771 100644 --- a/.gitlab/cc-env.sh +++ b/.gitlab/cc-env.sh @@ -17,6 +17,7 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\ :$DEPS_INSTALL_DIR/llvm-install/lib\ :$DEPS_INSTALL_DIR/libtool-install/lib\ :$DEPS_INSTALL_DIR/postgresql-install/lib\ +:$DEPS_INSTALL_DIR/python-install/lib\ :$LD_LIBRARY_PATH export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\ From 3e88b280da3e13b7512e30db82833a31314b93ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Fri, 29 Jan 2021 21:25:42 +0100 Subject: [PATCH 19/39] Load Boost::Python only for pythonparser. --- CMakeLists.txt | 2 +- plugins/python/parser/CMakeLists.txt | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e71ab356..d4e81c7d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ include(Functions.cmake) # Do some sanity check on the testing setup and enable testing if applicable. include(Testing.cmake) -find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread python) +find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread) find_package(Java REQUIRED) find_package(Odb REQUIRED) find_package(Threads REQUIRED) diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 991d1e9d0..4190338fb 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(PythonLibs REQUIRED) +find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread python) include_directories(${PYTHON_INCLUDE_DIRS}) @@ -12,8 +13,6 @@ include_directories( add_library(pythonparser SHARED src/pythonparser.cpp) -find_package (Python) - target_link_libraries(pythonparser util model From 976445c745177109aee1d45f85839572b90624a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Fri, 12 Mar 2021 00:47:01 +0100 Subject: [PATCH 20/39] Add correct libgit2 path to LD_LIBRARY_PATH for Ubuntu. Add postgreql bin folder to PATH. --- .gitlab/cc-env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/cc-env.sh b/.gitlab/cc-env.sh index ddf0e5771..39ef15adc 100644 --- a/.gitlab/cc-env.sh +++ b/.gitlab/cc-env.sh @@ -8,6 +8,7 @@ DEPS_INSTALL_DIR=$ROOT_DIR/deps-runtime CC_INSTALL_DIR=$ROOT_DIR/cc-install export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\ +:$DEPS_INSTALL_DIR/libgit2-install/lib\ :$DEPS_INSTALL_DIR/openssl-install/lib\ :$DEPS_INSTALL_DIR/graphviz-install/lib\ :$DEPS_INSTALL_DIR/libmagic-install/lib\ @@ -19,11 +20,13 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\ :$DEPS_INSTALL_DIR/postgresql-install/lib\ :$DEPS_INSTALL_DIR/python-install/lib\ :$LD_LIBRARY_PATH +# Note: libgit2-install/lib required on Ubuntu; libgit2-install/lib64 on SUSE export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\ :$DEPS_INSTALL_DIR/ctags-install/bin\ :$DEPS_INSTALL_DIR/python-install/bin\ :$DEPS_INSTALL_DIR/node-install/bin\ +:$DEPS_INSTALL_DIR/postgresql-install/bin\ :$CC_INSTALL_DIR/bin\ :$PATH From 980c3591779261870c21692086eaa0d200bc8a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Fri, 12 Mar 2021 03:17:13 +0100 Subject: [PATCH 21/39] Use FindPython3 instead of deprecated FindPythonLibs. --- plugins/python/parser/CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 4190338fb..00a59fb63 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -1,14 +1,13 @@ -find_package(PythonLibs REQUIRED) +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) find_package(Boost REQUIRED COMPONENTS filesystem log program_options regex system thread python) -include_directories(${PYTHON_INCLUDE_DIRS}) - include_directories( include ${CMAKE_SOURCE_DIR}/util/include ${CMAKE_SOURCE_DIR}/model/include ${CMAKE_SOURCE_DIR}/parser/include - ${PLUGIN_DIR}/model/include) + ${PLUGIN_DIR}/model/include + ${Python3_INCLUDE_DIRS}) add_library(pythonparser SHARED src/pythonparser.cpp) @@ -17,7 +16,7 @@ target_link_libraries(pythonparser util model pythonmodel - ${PYTHON_LIBRARIES} + ${Python3_LIBRARIES} ${Boost_LIBRARIES}) target_compile_options(pythonparser PUBLIC -Wno-unknown-pragmas) From 048e997e983885ed642ccd7a00e18a88efe4c4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Tue, 20 Apr 2021 18:41:06 +0200 Subject: [PATCH 22/39] Added RTLD_GLOBAL to dlopen. --- util/src/dynamiclibrary.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/dynamiclibrary.cpp b/util/src/dynamiclibrary.cpp index 251af2268..4d42c64ef 100644 --- a/util/src/dynamiclibrary.cpp +++ b/util/src/dynamiclibrary.cpp @@ -37,7 +37,7 @@ DynamicLibrary::DynamicLibrary(const std::string& path_) throw std::runtime_error(ss.str()); } #else - _handle = ::dlopen(path_.c_str(), RTLD_NOW); + _handle = ::dlopen(path_.c_str(), RTLD_NOW | RTLD_GLOBAL); if (!_handle) { const char *dlError = ::dlerror(); From fe6ecf55c3513715f29216de537874374a2089df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Tue, 20 Apr 2021 18:42:59 +0200 Subject: [PATCH 23/39] Set PYTHONHOME in cc-env to properly handle multiple Python installations when using CodeCompass. --- .gitlab/cc-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/cc-env.sh b/.gitlab/cc-env.sh index 39ef15adc..ba23ddf5e 100644 --- a/.gitlab/cc-env.sh +++ b/.gitlab/cc-env.sh @@ -32,6 +32,7 @@ export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\ export MAGIC=$DEPS_INSTALL_DIR/libmagic-install/share/misc/magic.mgc export JAVA_HOME=$DEPS_INSTALL_DIR/jdk-install +export PYTHONHOME=$DEPS_INSTALL_DIR/python-install if [ -f $ROOT_DIR/ccdb-tool/venv/bin/activate ]; then source $ROOT_DIR/ccdb-tool/venv/bin/activate From f2b9c21141eacf3dcfbf6c20a5e5a0dbad8f7f9c Mon Sep 17 00:00:00 2001 From: Bence Date: Wed, 21 Apr 2021 21:40:41 +0200 Subject: [PATCH 24/39] Handle Python 3.9 subscript (ast.Index and ast.ExtSlice are deprecated) Fix declaration inside except block Handled starred variable in with statement --- .../src/scripts/cc_python_parser/logger.py | 4 +-- .../member_access_collector.py | 33 ++++++++++++------- .../persistence/persistence.py | 2 +- .../scripts/cc_python_parser/scope_manager.py | 8 +++-- .../cc_python_parser/symbol_collector.py | 11 +++++-- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/logger.py b/plugins/python/parser/src/scripts/cc_python_parser/logger.py index a4621e7e2..e1ae69a30 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/logger.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/logger.py @@ -5,12 +5,12 @@ logger = logging.getLogger('test') logger.setLevel(logging.DEBUG) -file_handler = logging.FileHandler(str(directory) + '/test.log', 'w') +file_handler = logging.FileHandler(str(directory) + '/test.log', 'w', encoding='utf-8') file_handler.setLevel(logging.DEBUG) logger.addHandler(file_handler) db_logger = logging.getLogger('db_test') db_logger.setLevel(logging.DEBUG) -db_file_handler = logging.FileHandler(str(directory) + '/db_test.log', 'w') +db_file_handler = logging.FileHandler(str(directory) + '/db_test.log', 'w', encoding='utf-8') db_file_handler.setLevel(logging.DEBUG) db_logger.addHandler(db_file_handler) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py index db629605b..5348bbd83 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py @@ -1,4 +1,5 @@ import ast +import sys from typing import Optional, List, Any, Union @@ -205,19 +206,29 @@ def collect_keywords(call: ast.Call) -> List: @staticmethod def collect_slice(subscript: ast.Subscript) -> List[Union[SubscriptData.Index, SubscriptData.Slice]]: - # TODO: in python 3.9 this must be changed (ast.Index, ast.ExtSlice are deprecated) sub_slice = [] - def process_slice(node: (ast.Index, ast.Slice, ast.ExtSlice)): - if isinstance(node, ast.Index): - sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) - elif isinstance(node, ast.Slice): - sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) - elif isinstance(node, ast.ExtSlice): # 3.9 -> ast.Tuple - for dim in node.dims: - process_slice(dim) - else: - assert False, "Unknown slice: " + str(type(node)) + # since python 3.9 ast.Index, ast.ExtSlice are deprecated + if sys.version_info[0] == 3 and sys.version_info[1] >= 9: + def process_slice(node: (ast.Name, ast.Slice)): + if isinstance(node, ast.Slice): + sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) + elif isinstance(node, ast.Tuple): + for elem in node.elts: + process_slice(elem) + else: + sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) + else: + def process_slice(node: (ast.Index, ast.Slice, ast.ExtSlice)): + if isinstance(node, ast.Index): + sub_slice.append(MemberAccessCollector.SubscriptData.Index(node)) + elif isinstance(node, ast.Slice): + sub_slice.append(MemberAccessCollector.SubscriptData.Slice(node.lower, node.upper, node.step)) + elif isinstance(node, ast.ExtSlice): + for dim in node.dims: + process_slice(dim) + else: + assert False, f"Unknown slice: str(type(node))" process_slice(subscript.slice) return sub_slice diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py index 55dc77448..8dfd63ee6 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py @@ -11,7 +11,7 @@ class ModelPersistence: def __init__(self, c_persistence): self.c_persistence = c_persistence self.check_c_persistence() - self.log = open(str(PurePath(__file__).parent.joinpath('persistence_log.txt')), 'w+') + self.log = open(str(PurePath(__file__).parent.joinpath('persistence_log.txt')), 'w+', encoding='utf-8') def __del__(self): self.log.close() diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py index 46f4e7add..0f4e5f083 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py @@ -112,6 +112,7 @@ def append_module_variable_to_current_scope(self, module_variable: ModuleVariabl def is_same_declaration(var: VariableDeclaration) -> bool: return isinstance(var, ModuleVariableDeclaration) and \ var.imported_module_location == module_variable.imported_module_location + scope = self.get_current_lifetime_scope() if len([v for v in scope.variable_declarations if is_same_declaration(v)]) == 0: scope.append_variable(module_variable) @@ -154,12 +155,12 @@ def is_declaration_in_current_class(self, name: str) -> bool: def get_declaration(self, name: str) -> Optional[Union[Declaration, PlaceholderType]]: for scope in self.reverse(): if isinstance(scope, ClassScope) and scope is not self.get_current_lifetime_scope(): - continue # members can be accessed via self, except from the current class scope + continue # members can be accessed via self, except from the current class scope declaration = self.get_declaration_from_scope(name, scope) if declaration is not None: from cc_python_parser.variable_data import TypeVariableDeclaration if isinstance(declaration, TypeVariableDeclaration): - pass # print("NEED FIX: TypeVariable") + pass # print("NEED FIX: TypeVariable") return declaration current_class_scope = self.get_current_class_scope() if current_class_scope is not None: @@ -403,7 +404,8 @@ def get_class_declaration_from_scope(name: str, scope: Scope) -> Optional[ClassD def get_current_class_declaration(self) -> Optional[ClassDeclaration]: current_class_scope_found = False for scope in self.reverse(): - if current_class_scope_found and isinstance(scope, LifetimeScope): + if current_class_scope_found and isinstance(scope, LifetimeScope) and \ + not isinstance(scope, PartialLifetimeScope): return scope.class_declarations[-1] elif not current_class_scope_found: current_class_scope_found = isinstance(scope, ClassScope) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py index b73af54b2..a15c8c431 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py @@ -3,7 +3,7 @@ from typing import List, Optional, Any, Union, Set, TypeVar from cc_python_parser.built_in_functions import get_built_in_function -from cc_python_parser.built_in_types import GenericType, BuiltIn +from cc_python_parser.built_in_types import GenericType, BuiltIn, List as BuiltInList from cc_python_parser.common.file_position import FilePosition from cc_python_parser.common.utils import create_range_from_ast_node from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory @@ -14,7 +14,6 @@ from cc_python_parser.class_data import ClassDeclaration, ImportedClassDeclaration from cc_python_parser.class_preprocessor import PreprocessedClass from cc_python_parser.common.parser_tree import ParserTree -from cc_python_parser.common.position import Range from cc_python_parser.import_finder import ImportFinder from cc_python_parser.import_preprocessor import ImportTable from cc_python_parser.logger import logger @@ -696,7 +695,13 @@ def handle_vars(variables, types): for elem in variables.elts: handle_vars(elem, types) elif isinstance(variables, (ast.Attribute, ast.Subscript)): - pass # TODO: handle ast.Attribute and ast.Subscript eg: Lib > test > test_buffer.py + pass # TODO: handle ast.Attribute and ast.Subscript eg: Lib > test > test_buffer.py + elif isinstance(variables, ast.Starred): + r = create_range_from_ast_node(variables) + t = {BuiltInList(types)} + qn = self.scope_manager.get_qualified_name_from_current_scope(variables.value.id, r.start_position.line) + new_variable = VariableDeclaration(variables.value.id, qn, FilePosition(self.current_file, r), t) + self.scope_manager.append_variable_to_current_scope(new_variable) else: assert False From dfcf3e5b6639acfae8c10df6717e0758f6dca254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Tue, 13 Apr 2021 21:53:18 +0200 Subject: [PATCH 25/39] Fix libmagic download path in tarball building script It seems like the original source has gone. The project https://github.com/threatstack/libmagic/ was deleted or made private. --- .gitlab/build-deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/build-deps.sh b/.gitlab/build-deps.sh index 74c2b8660..e93378046 100644 --- a/.gitlab/build-deps.sh +++ b/.gitlab/build-deps.sh @@ -89,8 +89,8 @@ rm -f $PACKAGES_DIR/openssl-1.1.1h.tar.gz ############ cd $PACKAGES_DIR -wget --no-verbose --no-clobber https://github.com/threatstack/libmagic/archive/5.18.tar.gz -tar -xf 5.18.tar.gz +wget --no-verbose --no-clobber https://skynet.elte.hu/libmagic-5.18.tar.gz +tar -xf libmagic-5.18.tar.gz cd libmagic-5.18/ ./configure --quiet --prefix=$DEPS_INSTALL_RUNTIME_DIR/libmagic-install From d7714125a2f5d7e9071ec11d766c1c8d6c940e8c Mon Sep 17 00:00:00 2001 From: Bence Date: Sun, 16 May 2021 16:58:59 +0200 Subject: [PATCH 26/39] Handle symlinks in python parser. --- .../src/scripts/cc_python_parser/file_info.py | 13 ++++-- .../src/scripts/cc_python_parser/parser.py | 42 ++++++++----------- .../cc_python_parser/persistence/__init__.py | 2 +- .../cc_python_parser/persistence/class_dto.py | 15 +++---- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py index 112cf6422..bfa10e0f5 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py @@ -18,13 +18,18 @@ class ProcessStatus(Enum): # TODO: expr_1; import; expr_2; - correct script -> not a problem class FileInfo: - def __init__(self, file: str, path: Optional[PurePath]): # TODO: remove optional - self.file: str = file - self.path: Optional[PurePath] = path + def __init__(self, path: PurePath): + self.path: PurePath = path self.symbol_collector: Optional[SymbolCollector] = None self.preprocessed_file: PreprocessedFile = PreprocessedFile(path) self.status: ProcessStatus = ProcessStatus.WAITING + def get_file(self): + return self.path.name + + def get_file_name(self): + return self.path.stem + def preprocess_file(self, tree) -> None: self.preprocessed_file.visit(tree) self.status = ProcessStatus.PREPROCESSED @@ -36,7 +41,7 @@ def set_variable_collector(self, variable_collector: SymbolCollector) -> None: def create_dto(self) -> FileDTO: file_dto = FileDTO() file_dto.path = str(self.path) - file_dto.file_name = self.file + file_dto.file_name = self.path.name file_dto.timestamp = Path(self.path).stat().st_mtime file_dto.content = FileContentDTO(self.get_content(self.path)) file_dto.parent = str(self.path.parent) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parser.py b/plugins/python/parser/src/scripts/cc_python_parser/parser.py index 5c586500e..705411990 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/parser.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/parser.py @@ -1,7 +1,7 @@ import ast import os import sys -from pathlib import PurePath +from pathlib import PurePath, Path from typing import List, Optional, Union, Set, Tuple from cc_python_parser.common.parser_tree import ParserTree @@ -48,23 +48,15 @@ def collect_files(self): self.process_directory(PurePath(directory)) def process_directory(self, directory: PurePath): - sub_directories = [] - for element in os.listdir(str(directory)): - path = directory.joinpath(element) - if os.path.isfile(str(path)): - if self.exceptions.is_file_exception(path): - continue - if element.endswith(".py"): - self.files.append(FileInfo(element, path)) - elif os.path.isdir(str(path)): - if self.exceptions.is_dictionary_exception(path): - continue - sub_directories.append(path) - else: - assert False, "Unknown element (file, directory): " + str(path) - - for subdir in sub_directories: - self.process_directory(subdir) + for root, folders, files in os.walk(directory, followlinks=True): + for file in files: + file_path = Path(root, file) + if file_path.is_symlink(): + if not file_path.exists(): + continue + file_path = file_path.resolve() + if file_path.suffix == '.py': + self.files.append(FileInfo(file_path)) def parse(self) -> None: metrics.start_parsing() @@ -77,11 +69,11 @@ def parse(self) -> None: def parse_file(self, file_info: FileInfo) -> None: global current_file - current_file = file_info.file + current_file = file_info.get_file() if file_info.status != ProcessStatus.WAITING or current_file[-3:] != '.py': return - logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - db_logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") + db_logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") if file_info.path is None: return @@ -115,10 +107,10 @@ def parse_file(self, file_info: FileInfo) -> None: elif len(dependency_file_info) == 1: if dependency_file_info[0].status is ProcessStatus.WAITING: self.parse_file(dependency_file_info[0]) - current_file = file_info.file - logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + current_file = file_info.get_file() + logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") db_logger. \ - debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") else: assert False, 'Multiple file occurrence: ' + dependency.get_file() sc = SymbolCollector(ParserTree(tree), file_info.path, file_info.preprocessed_file, @@ -138,7 +130,7 @@ def handle_modules_outside_of_project(self, file_info: FileInfo) -> None: if not any(PurePath(mm) in m.parents for mm in self.directories) and \ m not in self.other_modules: self.other_modules.add(m) - new_file_info = FileInfo(m.name, m) + new_file_info = FileInfo(m) self.other_module_files.append(new_file_info) self.parse_file(new_file_info) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py index f15edfb7d..a435651d9 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py @@ -1,2 +1,2 @@ __all__ = ["build_action", "build_source_target", "file_dto", "file_content_dto", "persistence", "class_dto", - "function_dto", "import_dto.py", "variable_dto", "base_dto"] + "function_dto", "import_dto", "variable_dto", "base_dto"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py index 4fc74f0c1..ea8263929 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py @@ -10,14 +10,15 @@ class ClassDeclarationDTO(DeclarationDTO, DocumentedDTO): class ClassMembersDTO: def __init__(self): - self.methods: List[FunctionDTO] = [] - self.static_methods: List[FunctionDTO] = [] - self.attributes: List[VariableDTO] = [] - self.static_attributes: List[VariableDTO] = [] + self.methods: List[FunctionDeclarationDTO] = [] + self.static_methods: List[FunctionDeclarationDTO] = [] + self.attributes: List[VariableDeclarationDTO] = [] + self.static_attributes: List[VariableDeclarationDTO] = [] self.classes: List[ClassDeclarationDTO] = [] - - def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], - usages: List[UsageDTO], base_classes: Set[str], members: 'ClassDeclarationDTO.ClassMembersDTO', documentation: str): + + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, + types: Set[str], usages: List[UsageDTO], base_classes: Set[str], + members: 'ClassDeclarationDTO.ClassMembersDTO', documentation: str): DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) DocumentedDTO.__init__(self, documentation) self.base_classes: List[str] = list(base_classes) From 0ee367fa1a94085a8674b300d4d0276d73a065a9 Mon Sep 17 00:00:00 2001 From: Bence Date: Sun, 16 May 2021 16:58:59 +0200 Subject: [PATCH 27/39] Handle symlinks in python parser. --- .../src/scripts/cc_python_parser/file_info.py | 13 ++++-- .../src/scripts/cc_python_parser/parser.py | 42 ++++++++----------- .../cc_python_parser/persistence/__init__.py | 2 +- .../cc_python_parser/persistence/class_dto.py | 15 +++---- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py index 112cf6422..bfa10e0f5 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py @@ -18,13 +18,18 @@ class ProcessStatus(Enum): # TODO: expr_1; import; expr_2; - correct script -> not a problem class FileInfo: - def __init__(self, file: str, path: Optional[PurePath]): # TODO: remove optional - self.file: str = file - self.path: Optional[PurePath] = path + def __init__(self, path: PurePath): + self.path: PurePath = path self.symbol_collector: Optional[SymbolCollector] = None self.preprocessed_file: PreprocessedFile = PreprocessedFile(path) self.status: ProcessStatus = ProcessStatus.WAITING + def get_file(self): + return self.path.name + + def get_file_name(self): + return self.path.stem + def preprocess_file(self, tree) -> None: self.preprocessed_file.visit(tree) self.status = ProcessStatus.PREPROCESSED @@ -36,7 +41,7 @@ def set_variable_collector(self, variable_collector: SymbolCollector) -> None: def create_dto(self) -> FileDTO: file_dto = FileDTO() file_dto.path = str(self.path) - file_dto.file_name = self.file + file_dto.file_name = self.path.name file_dto.timestamp = Path(self.path).stat().st_mtime file_dto.content = FileContentDTO(self.get_content(self.path)) file_dto.parent = str(self.path.parent) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parser.py b/plugins/python/parser/src/scripts/cc_python_parser/parser.py index 5c586500e..705411990 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/parser.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/parser.py @@ -1,7 +1,7 @@ import ast import os import sys -from pathlib import PurePath +from pathlib import PurePath, Path from typing import List, Optional, Union, Set, Tuple from cc_python_parser.common.parser_tree import ParserTree @@ -48,23 +48,15 @@ def collect_files(self): self.process_directory(PurePath(directory)) def process_directory(self, directory: PurePath): - sub_directories = [] - for element in os.listdir(str(directory)): - path = directory.joinpath(element) - if os.path.isfile(str(path)): - if self.exceptions.is_file_exception(path): - continue - if element.endswith(".py"): - self.files.append(FileInfo(element, path)) - elif os.path.isdir(str(path)): - if self.exceptions.is_dictionary_exception(path): - continue - sub_directories.append(path) - else: - assert False, "Unknown element (file, directory): " + str(path) - - for subdir in sub_directories: - self.process_directory(subdir) + for root, folders, files in os.walk(directory, followlinks=True): + for file in files: + file_path = Path(root, file) + if file_path.is_symlink(): + if not file_path.exists(): + continue + file_path = file_path.resolve() + if file_path.suffix == '.py': + self.files.append(FileInfo(file_path)) def parse(self) -> None: metrics.start_parsing() @@ -77,11 +69,11 @@ def parse(self) -> None: def parse_file(self, file_info: FileInfo) -> None: global current_file - current_file = file_info.file + current_file = file_info.get_file() if file_info.status != ProcessStatus.WAITING or current_file[-3:] != '.py': return - logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') - db_logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") + db_logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") if file_info.path is None: return @@ -115,10 +107,10 @@ def parse_file(self, file_info: FileInfo) -> None: elif len(dependency_file_info) == 1: if dependency_file_info[0].status is ProcessStatus.WAITING: self.parse_file(dependency_file_info[0]) - current_file = file_info.file - logger.debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + current_file = file_info.get_file() + logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") db_logger. \ - debug('\nFILE: ' + file_info.file[:-3] + '\n=========================================') + debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") else: assert False, 'Multiple file occurrence: ' + dependency.get_file() sc = SymbolCollector(ParserTree(tree), file_info.path, file_info.preprocessed_file, @@ -138,7 +130,7 @@ def handle_modules_outside_of_project(self, file_info: FileInfo) -> None: if not any(PurePath(mm) in m.parents for mm in self.directories) and \ m not in self.other_modules: self.other_modules.add(m) - new_file_info = FileInfo(m.name, m) + new_file_info = FileInfo(m) self.other_module_files.append(new_file_info) self.parse_file(new_file_info) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py index f15edfb7d..a435651d9 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/__init__.py @@ -1,2 +1,2 @@ __all__ = ["build_action", "build_source_target", "file_dto", "file_content_dto", "persistence", "class_dto", - "function_dto", "import_dto.py", "variable_dto", "base_dto"] + "function_dto", "import_dto", "variable_dto", "base_dto"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py index 4fc74f0c1..ea8263929 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/class_dto.py @@ -10,14 +10,15 @@ class ClassDeclarationDTO(DeclarationDTO, DocumentedDTO): class ClassMembersDTO: def __init__(self): - self.methods: List[FunctionDTO] = [] - self.static_methods: List[FunctionDTO] = [] - self.attributes: List[VariableDTO] = [] - self.static_attributes: List[VariableDTO] = [] + self.methods: List[FunctionDeclarationDTO] = [] + self.static_methods: List[FunctionDeclarationDTO] = [] + self.attributes: List[VariableDeclarationDTO] = [] + self.static_attributes: List[VariableDeclarationDTO] = [] self.classes: List[ClassDeclarationDTO] = [] - - def __init__(self, name: str, qualified_name: str, file_position: FilePosition, types: Set[str], - usages: List[UsageDTO], base_classes: Set[str], members: 'ClassDeclarationDTO.ClassMembersDTO', documentation: str): + + def __init__(self, name: str, qualified_name: str, file_position: FilePosition, + types: Set[str], usages: List[UsageDTO], base_classes: Set[str], + members: 'ClassDeclarationDTO.ClassMembersDTO', documentation: str): DeclarationDTO.__init__(self, name, qualified_name, file_position, types, usages) DocumentedDTO.__init__(self, documentation) self.base_classes: List[str] = list(base_classes) From e4ce9b6a8b342e6136d7b8ba1b0afff8bdf7da82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Sun, 9 May 2021 11:01:09 +0200 Subject: [PATCH 28/39] Tarball CI: updated outdated Boost download URL (cherry picked from commit cd7d31936872730bb8b054c85c7e9736cb841918) --- .gitlab/build-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/build-deps.sh b/.gitlab/build-deps.sh index e93378046..d2bae225c 100644 --- a/.gitlab/build-deps.sh +++ b/.gitlab/build-deps.sh @@ -310,7 +310,7 @@ rm -f $PACKAGES_DIR/ctags-p5.9.20201129.0.tar.gz if [ ! -f $DEPS_INSTALL_RUNTIME_DIR/boost-install/lib/libboost_program_options.so ]; then cd $PACKAGES_DIR - wget --no-verbose --no-clobber https://dl.bintray.com/boostorg/release/1.74.0/source/boost_1_74_0.tar.gz + wget --no-verbose --no-clobber https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source/boost_1_74_0.tar.gz tar -xf boost_1_74_0.tar.gz cd boost_1_74_0 From 304427eafd28e2ad1bbee3064bb2a7c8b9aef287 Mon Sep 17 00:00:00 2001 From: Bence Date: Wed, 28 Jul 2021 21:15:09 +0200 Subject: [PATCH 29/39] Use file and directory exceptions. Introduce process_file_content. --- .../scripts/cc_python_parser/common/utils.py | 22 ++++++++++++- .../src/scripts/cc_python_parser/file_info.py | 19 +++++------ .../cc_python_parser/import_preprocessor.py | 6 ++++ .../src/scripts/cc_python_parser/parser.py | 33 +++++++++---------- .../cc_python_parser/symbol_collector.py | 2 +- 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py index 4b7d0004a..8f680a204 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py @@ -1,5 +1,6 @@ import ast -from typing import Final, List +from pathlib import PurePath, Path +from typing import Final, List, Callable from cc_python_parser.common.position import Range, Position @@ -7,6 +8,25 @@ ENCODINGS: Final[List[str]] = ['utf-8', 'iso-8859-1'] +def process_file_content(path: PurePath, func: Callable[[str, int], None]) -> None: + if not Path(path).exists(): + return + for encoding in ENCODINGS: + try: + with open(str(path), "r", encoding=encoding) as source: + content = source.read() + source.seek(0) + func(content, len(source.readlines())) + except UnicodeDecodeError: + continue + except FileNotFoundError: + print(f"File not found: {path}") + continue + else: + return + print(f"Unhandled encoding in {str(path)}") + + def has_attr(obj, attrs) -> bool: for attr in attrs.split("."): if hasattr(obj, attr): diff --git a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py index bfa10e0f5..a1f1d1685 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py @@ -2,7 +2,7 @@ from pathlib import PurePath, Path from typing import Optional -from cc_python_parser.common.utils import ENCODINGS +from cc_python_parser.common.utils import process_file_content from cc_python_parser.persistence.file_content_dto import FileContentDTO from cc_python_parser.persistence.file_dto import FileDTO from cc_python_parser.preprocessed_file import PreprocessedFile @@ -51,15 +51,14 @@ def create_dto(self) -> FileDTO: @staticmethod def get_content(file: PurePath) -> str: - for e in ENCODINGS: - try: - with open(str(file), "r", encoding=e) as f: - content = f.read() - except UnicodeDecodeError: - pass - else: - return content - assert False, f"Unhandled coding in {str(file)}" + content = "" + + def handle_file_content(c, _): + nonlocal content + content = c + + process_file_content(file, handle_file_content) + return content @staticmethod def get_parse_status(status: ProcessStatus) -> int: diff --git a/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py index 79455121e..9dab36bf6 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py @@ -81,6 +81,8 @@ def append_import_from(self, node: ast.ImportFrom, is_local: bool = False): pass # print() except (AttributeError, ModuleNotFoundError, ImportError, ValueError): pass + except KeyError: + return finally: if m is None: try: @@ -88,6 +90,8 @@ def append_import_from(self, node: ast.ImportFrom, is_local: bool = False): m = util.find_spec(node.module) except (AttributeError, ModuleNotFoundError, ImportError, ValueError): pass + except KeyError: + return if m is not None and m.origin is not None: self.import_paths.add(PurePath(m.origin)) @@ -117,6 +121,8 @@ def convert_ast_import_from_to_import(node: ast.ImportFrom, is_local: bool = Fal except (AttributeError, ModuleNotFoundError, ImportError, ValueError): # v3.7: before: AttributeError, after: ModuleNotFoundError is_module = False + except KeyError: + return [] imports = [] r = create_range_from_ast_node(node) if is_module: # modules diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parser.py b/plugins/python/parser/src/scripts/cc_python_parser/parser.py index 705411990..81a3c219b 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/parser.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/parser.py @@ -5,7 +5,7 @@ from typing import List, Optional, Union, Set, Tuple from cc_python_parser.common.parser_tree import ParserTree -from cc_python_parser.common.utils import ENCODINGS +from cc_python_parser.common.utils import process_file_content from cc_python_parser.file_info import FileInfo, ProcessStatus from cc_python_parser.function_symbol_collector import FunctionSymbolCollector from cc_python_parser.function_symbol_collector_factory import FunctionSymbolCollectorFactory @@ -48,8 +48,11 @@ def collect_files(self): self.process_directory(PurePath(directory)) def process_directory(self, directory: PurePath): - for root, folders, files in os.walk(directory, followlinks=True): + for root, dirs, files in os.walk(directory, topdown=True, followlinks=True): + dirs[:] = [d for d in dirs if not self.exceptions.is_dictionary_exception(d)] for file in files: + if self.exceptions.is_file_exception(file): + continue file_path = Path(root, file) if file_path.is_symlink(): if not file_path.exists(): @@ -72,29 +75,23 @@ def parse_file(self, file_info: FileInfo) -> None: current_file = file_info.get_file() if file_info.status != ProcessStatus.WAITING or current_file[-3:] != '.py': return - logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") - db_logger.debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") + logger.debug('\nFILE: ' + file_info.get_file_name() + '\n=========================================') + db_logger.debug('\nFILE: ' + file_info.get_file_name() + '\n=========================================') if file_info.path is None: return tree = None - for e in ENCODINGS: + + def handle_file_content(c, line_num): + nonlocal tree try: - with open(str(file_info.path), "r", encoding=e) as source: - t = ast.parse(source.read()) - source.seek(0) - metrics.add_line_count(len(source.readlines())) - metrics.add_file_count() - except UnicodeDecodeError: - pass - except FileNotFoundError: - print(f"File not found: {file_info.path}") + tree = ast.parse(c) + metrics.add_line_count(line_num) + metrics.add_file_count() except SyntaxError as e: print(f"Syntax error in file {e.filename} at (line - {e.lineno}, column - {e.offset}): {e.text}") - break - else: - tree = t - break + + process_file_content(file_info.path, handle_file_content) if tree is None: return diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py index a15c8c431..f7d3d40b4 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py @@ -827,7 +827,7 @@ def visit_Name(self, node: ast.Name) -> Any: get_qualified_name_from_current_scope(node.id, r.start_position.line) self.scope_manager.append_variable_to_current_scope( VariableDeclaration(node.id, qualified_name, FilePosition(self.current_file, r))) - assert False, 'Assignment target, was not handled at ' + str(r) + assert False, f'Assignment target, was not handled at {str(r)} in {self.current_file}' elif isinstance(node.ctx, (ast.Load, ast.Del)): self.append_variable_usage(node.id, node) else: From d09fd152943f4615733ae2abb169e67e2141e462 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 31 Oct 2021 18:22:54 +0100 Subject: [PATCH 30/39] Use sets instead of vector. Remove unnecessary inner class usage persisting. --- plugins/python/parser/src/pythonparser.cpp | 234 ++++++++++++++------- 1 file changed, 154 insertions(+), 80 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 50f554121..3110005b2 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -67,19 +68,21 @@ class Persistence model::PythonEntityPtr getPythonEntity(const std::string& qualifiedName); model::PythonClassPtr getPythonClass(const std::string& qualifiedName); model::PythonVariablePtr getPythonVariable(const std::string& qualifiedName); - - std::vector _astNodes; - std::vector _variables; - std::map> _variableUsages; - std::vector _functions; - std::map> _functionUsages; - std::vector _classes; - std::map> _classUsages; - std::vector _members; - std::vector _inheritance; - std::vector _imports; - std::vector _documentations; - std::vector _types; + bool isAstNodePersisted(const model::PythonAstNodePtr& node) const; + bool isAstNodePersisted(const std::set& nodes, const model::PythonAstNodePtr& node) const; + + std::set _astNodes; + std::set _variables; + std::map> _variableUsages; + std::set _functions; + std::map> _functionUsages; + std::set _classes; + std::map> _classUsages; + std::set _members; + std::set _inheritance; + std::set _imports; + std::set _documentations; + std::set _types; }; template @@ -96,24 +99,36 @@ Persistence::~Persistence() ctx.srcMgr.persistFiles(); (util::OdbTransaction(ctx.db))([this]{ - util::persistAll(_astNodes, ctx.db); + std::vector tmp(_astNodes.begin(), _astNodes.end()); + util::persistAll(tmp, ctx.db); for(auto& ast : _variableUsages){ - util::persistAll(ast.second, ctx.db); + std::vector v(ast.second.begin(), ast.second.end()); + util::persistAll(v, ctx.db); } for(auto& ast : _functionUsages){ - util::persistAll(ast.second, ctx.db); + std::vector v(ast.second.begin(), ast.second.end()); + util::persistAll(v, ctx.db); } for(auto& ast : _classUsages){ - util::persistAll(ast.second, ctx.db); + std::vector v(ast.second.begin(), ast.second.end()); + util::persistAll(v, ctx.db); } - util::persistAll(_variables, ctx.db); - util::persistAll(_functions, ctx.db); - util::persistAll(_classes, ctx.db); - util::persistAll(_members, ctx.db); - util::persistAll(_inheritance, ctx.db); - util::persistAll(_imports, ctx.db); - util::persistAll(_documentations, ctx.db); - util::persistAll(_types, ctx.db); + std::vector tmp_variables(_variables.begin(), _variables.end()); + util::persistAll(tmp_variables, ctx.db); + std::vector tmp_functions(_functions.begin(), _functions.end()); + util::persistAll(tmp_functions, ctx.db); + std::vector tmp_classes(_classes.begin(), _classes.end()); + util::persistAll(tmp_classes, ctx.db); + std::vector tmp_members(_members.begin(), _members.end()); + util::persistAll(tmp_members, ctx.db); + std::vector tmp_inheritance(_inheritance.begin(), _inheritance.end()); + util::persistAll(tmp_inheritance, ctx.db); + std::vector tmp_imports(_imports.begin(), _imports.end()); + util::persistAll(tmp_imports, ctx.db); + std::vector tmp_documentations(_documentations.begin(), _documentations.end()); + util::persistAll(tmp_documentations, ctx.db); + std::vector tmp_types(_types.begin(), _types.end()); + util::persistAll(tmp_types, ctx.db); }); } @@ -205,7 +220,11 @@ void Persistence::persistVariable(boost::python::object pyVariable) varAstNode->id = model::createIdentifier(*varAstNode); - _astNodes.push_back(varAstNode); + if(isAstNodePersisted(varAstNode)){ + return; + } + + _astNodes.insert(varAstNode); model::PythonVariablePtr variable(new model::PythonVariable); variable->astNodeId = varAstNode->id; variable->name = boost::python::extract(name); @@ -213,7 +232,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) variable->visibility = boost::python::extract(visibility); variable->id = model::createIdentifier(*variable); - _variables.push_back(variable); + _variables.insert(variable); for(int i = 0; itype = t->id; type->symbol = variable->id; - _types.push_back(type); + _types.insert(type); } _variableUsages[variable->id] = {}; @@ -242,7 +261,11 @@ void Persistence::persistVariable(boost::python::object pyVariable) usageAstNode->id = model::createIdentifier(*usageAstNode); - _variableUsages[variable->id].push_back(usageAstNode); + if(isAstNodePersisted(_variableUsages[variable->id], usageAstNode)) { + continue; + } + + _variableUsages[variable->id].insert(usageAstNode); } } catch (const odb::object_already_persistent& ex) @@ -288,7 +311,11 @@ void Persistence::persistFunction(boost::python::object pyFunction) funcAstNode->id = model::createIdentifier(*funcAstNode); - _astNodes.push_back(funcAstNode); + if(isAstNodePersisted(funcAstNode)){ + return; + } + + _astNodes.insert(funcAstNode); model::PythonFunctionPtr function(new model::PythonFunction); function->astNodeId = funcAstNode->id; @@ -314,14 +341,14 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->locals.push_back(local); } - _functions.push_back(function); + _functions.insert(function); model::PythonDocumentationPtr documentation(new model::PythonDocumentation); documentation->documentation = boost::python::extract(pyDocumentation); documentation->documented = function->id; documentation->documentationKind = model::PythonDocumentation::Function; - _documentations.push_back(documentation); + _documentations.insert(documentation); for(int i = 0; itype = t->id; type->symbol = function->id; - _types.push_back(type); + _types.insert(type); } _functionUsages[function->id] = {}; @@ -350,7 +377,11 @@ void Persistence::persistFunction(boost::python::object pyFunction) usageAstNode->id = model::createIdentifier(*usageAstNode); - _functionUsages[function->id].push_back(usageAstNode); + if(isAstNodePersisted(_functionUsages[function->id], usageAstNode)) { + continue; + } + + _functionUsages[function->id].insert(usageAstNode); } } catch(std::exception e){ std::cout << "Func exception:" << e.what() << std::endl; @@ -378,7 +409,11 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) classAstNode->id = model::createIdentifier(*classAstNode); - _astNodes.push_back(classAstNode); + if(isAstNodePersisted(classAstNode)){ + return; + } + + _astNodes.insert(classAstNode); model::PythonClassPtr cl(new model::PythonClass); cl->astNodeId = classAstNode->id; @@ -388,7 +423,7 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) cl->id = model::createIdentifier(*cl); - _classes.push_back(cl); + _classes.insert(cl); } catch(std::exception e){ std::cout << "Preprocessed class exception:" << e.what() << std::endl; @@ -424,8 +459,7 @@ void Persistence::persistClass(boost::python::object pyClass) documentation->documented = cl->id; documentation->documentationKind = model::PythonDocumentation::Class; - _documentations.push_back(documentation); - + _documentations.insert(documentation); _classUsages[cl->id] = {}; for(int i = 0; i usageFileLoc = @@ -441,17 +475,24 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - _classUsages[cl->id].push_back(usageAstNode); + if(isAstNodePersisted(_classUsages[cl->id], usageAstNode)) { + continue; + } + + _classUsages[cl->id].insert(usageAstNode); } for(int i = 0; iderived = cl->id; std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); + auto baseClass = getPythonEntity(baseClassQualifiedName); + if(baseClass == nullptr){ + continue; + } + inheritance->base = baseClass->id; - inheritance->base = getPythonEntity(baseClassQualifiedName)->id; - - _inheritance.push_back(inheritance); + _inheritance.insert(inheritance); } boost::python::list methods = boost::python::extract(members.attr("methods")); @@ -471,6 +512,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ std::cout << "method is none" << std::endl; + continue; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -478,10 +520,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; - _members.push_back(classMember); + _members.insert(classMember); boost::python::list _usages = boost::python::extract(methods[i].attr("usages")); - std::vector& _funcUsages = _functionUsages[method->id]; + std::set& _funcUsages = _functionUsages[method->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -498,7 +540,11 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - _funcUsages.push_back(usageAstNode); + if(isAstNodePersisted(_funcUsages, usageAstNode)) { + continue; + } + + _funcUsages.insert(usageAstNode); } } @@ -508,6 +554,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ std::cout << "static method is none" << std::endl; + continue; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -515,10 +562,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Method; classMember->staticMember = true; - _members.push_back(classMember); + _members.insert(classMember); boost::python::list _usages = boost::python::extract(staticMethods[i].attr("usages")); - std::vector& _funcUsages = _functionUsages[method->id]; + std::set& _funcUsages = _functionUsages[method->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -535,7 +582,11 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - _funcUsages.push_back(usageAstNode); + if(isAstNodePersisted(_funcUsages, usageAstNode)) { + continue; + } + + _funcUsages.insert(usageAstNode); } } @@ -548,6 +599,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ std::cout << "attr is none" << std::endl; + continue; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -555,10 +607,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = false; - _members.push_back(classMember); + _members.insert(classMember); boost::python::list _usages = boost::python::extract(attributes[i].attr("usages")); - std::vector& _varUsages = _variableUsages[attr->id]; + std::set& _varUsages = _variableUsages[attr->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -575,7 +627,11 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - _varUsages.push_back(usageAstNode); + if(isAstNodePersisted(_varUsages, usageAstNode)) { + continue; + } + + _varUsages.insert(usageAstNode); } } @@ -585,6 +641,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ std::cout << "static attr is none" << std::endl; + continue; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -592,10 +649,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = true; - _members.push_back(classMember); + _members.insert(classMember); boost::python::list _usages = boost::python::extract(staticAttributes[i].attr("usages")); - std::vector& _varUsages = _variableUsages[attr->id]; + std::set& _varUsages = _variableUsages[attr->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -612,7 +669,11 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - _varUsages.push_back(usageAstNode); + if(isAstNodePersisted(_varUsages, usageAstNode)) { + continue; + } + + _varUsages.insert(usageAstNode); } } @@ -622,6 +683,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr inner = getPythonEntity(qualifiedName); if(inner == nullptr){ std::cout << "inner class is none" << std::endl; + continue; } classMember->astNodeId = inner->astNodeId; classMember->memberId = inner->id; @@ -629,28 +691,7 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Class; classMember->staticMember = false; - _members.push_back(classMember); - - boost::python::list _usages = boost::python::extract(classes[i].attr("usages")); - std::vector& _clUsages = _classUsages[cl->id]; - for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); - if(_fl == boost::none || - std::find_if(_clUsages.begin(), _clUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _clUsages.end()) - { - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = _fl.get(); - usageAstNode->qualifiedName = qualifiedName; - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - _clUsages.push_back(usageAstNode); - } + _members.insert(classMember); } } catch(std::exception e){ std::cout << "Class exception:" << e.what() << std::endl; @@ -690,13 +731,17 @@ void Persistence::persistImport(boost::python::object pyImport) moduleAstNode->id = model::createIdentifier(*moduleAstNode); + if(isAstNodePersisted(moduleAstNode)){ + continue; + } + model::PythonImportPtr moduleImport(new model::PythonImport); moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; - _astNodes.push_back(moduleAstNode); - _imports.push_back(moduleImport); + _astNodes.insert(moduleAstNode); + _imports.insert(moduleImport); } boost::python::list importDict = importedSymbols.items(); @@ -720,18 +765,25 @@ void Persistence::persistImport(boost::python::object pyImport) moduleAstNode->id = model::createIdentifier(*moduleAstNode); + if(isAstNodePersisted(moduleAstNode)){ + continue; + } + for (int j = 0; j < boost::python::len(import[1]); ++j){ model::PythonImportPtr moduleImport(new model::PythonImport); moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; auto symb = getPythonEntity(boost::python::extract(import[1][j])); + if(symb == nullptr){ + continue; + } moduleImport->importedSymbol = symb->id; - _imports.push_back(moduleImport); + _imports.insert(moduleImport); } - _astNodes.push_back(moduleAstNode); + _astNodes.insert(moduleAstNode); } } catch(std::exception e){ std::cout << "Import exception:" << e.what() << std::endl; @@ -820,6 +872,28 @@ model::PythonClassPtr Persistence::getPythonClass(const std::string& qualifiedNa return nullptr; } +bool Persistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const +{ + for(auto it : _astNodes){ + if(*it == *node){ + return true; + } + } + + return false; +} + +bool Persistence::isAstNodePersisted(const std::set& nodes, const model::PythonAstNodePtr& node) const +{ + for(auto it : nodes){ + if(*it == *node){ + return true; + } + } + + return false; +} + typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ From 9b84d9be4719a404b723649cf2d350f03e2d0377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Sun, 21 Nov 2021 16:22:57 +0100 Subject: [PATCH 31/39] Tarball CI: per branch caching of dependencies --- .gitlab/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/ci.yml b/.gitlab/ci.yml index 248ca4c93..986b9a8a3 100644 --- a/.gitlab/ci.yml +++ b/.gitlab/ci.yml @@ -62,7 +62,7 @@ tarball suse-15: extends: .tarball image: opensuse/leap:15 cache: - key: "leap" + key: "leap-$CI_COMMIT_REF_SLUG" variables: GCC_VERSION: 9.3.0 ODB_VERSION: 2.5.0 @@ -85,7 +85,7 @@ tarball suse-42.1: extends: .tarball image: opensuse/archive:42.1 cache: - key: "malachite" + key: "malachite-$CI_COMMIT_REF_SLUG" variables: GCC_VERSION: 5.5.0 ODB_VERSION: 2.4.0 @@ -111,7 +111,7 @@ tarball ubuntu-16.04: extends: .tarball image: ubuntu:16.04 cache: - key: "xenial" + key: "xenial-$CI_COMMIT_REF_SLUG" variables: GCC_VERSION: 5.5.0 ODB_VERSION: 2.4.0 From 97d5b3ba287d34db0f6a2c6fc2a5ba01345cec33 Mon Sep 17 00:00:00 2001 From: Bence Date: Sat, 27 Nov 2021 14:08:08 +0100 Subject: [PATCH 32/39] Use config.json in main.py to parse projects. --- .../src/scripts/cc_python_parser/main.py | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/main.py b/plugins/python/parser/src/scripts/cc_python_parser/main.py index 43b54ecdc..19c34d2ed 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/main.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/main.py @@ -1,7 +1,9 @@ import os +import json from copy import copy -from pathlib import PurePath -from typing import Final +from json import JSONDecodeError +from pathlib import PurePath, Path +from typing import List, Final from cc_python_parser.common.metrics import metrics, PythonParserMetrics from cc_python_parser.parse_exception import ParseException @@ -9,6 +11,36 @@ from cc_python_parser.persistence.persistence import ModelPersistence +def read_config() -> List[str]: + """ + Reads projects to parse from 'config.json', if it exists. Projects can be skipped with the "status" key: + { + "projects": [ + { + "project": "project_path", + "status": true/false + } + ] + } + """ + + config_file_name: Final[str] = 'config.json' + project_roots = [] + if Path(config_file_name).exists(): + with open(config_file_name, 'r') as config_file: + try: + config = json.load(config_file) + except JSONDecodeError as e: + print(f"Invalid JSON file: config.json (line: {e.lineno}, column: {e.colno})") + return project_roots + except Exception as e: + print(f"Error during config.json decoding: {e}") + return project_roots + for project in config['projects']: + if project['status']: + project_roots.append(project['project']) + return project_roots + def main(): def directory_exception(path: PurePath) -> bool: directory = os.path.basename(os.path.normpath(str(path))) @@ -17,25 +49,7 @@ def directory_exception(path: PurePath) -> bool: def file_exception(path: PurePath) -> bool: return False - code_compass: Final[str] = "/home/rmfcnb/ELTE/Code-Compass-Python-Plugin" - code_checker: Final[str] = "/home/rmfcnb/ELTE/CodeChecker" - python_std: Final[str] = "/usr/lib/python3.8" - manim: Final[str] = "/home/rmfcnb/ELTE/manim" - numpy: Final[str] = "/home/rmfcnb/ELTE/numpy" - algo: Final[str] = "/home/rmfcnb/ELTE/TheAlgorithms" - tensorflow: Final[str] = "/home/rmfcnb/ELTE/tensorflow" - - test: Final[str] = "/home/rmfcnb/ELTE/Test" - - project_roots = [code_compass, - code_checker, - python_std, - manim, - numpy, - algo, - tensorflow] - - project_roots = [test] + project_roots = read_config() metrics_result = [] for pr in project_roots: @@ -53,8 +67,6 @@ def file_exception(path: PurePath) -> bool: print(all_metrics) - pass - if __name__ == '__main__': main() From 811470ce5103938b27fdc3e1d86dc9de10f5b995 Mon Sep 17 00:00:00 2001 From: Bence Date: Sat, 27 Nov 2021 14:15:52 +0100 Subject: [PATCH 33/39] Use qualified name in imports too. --- .../src/scripts/cc_python_parser/base_data.py | 4 +-- .../scripts/cc_python_parser/class_data.py | 4 +-- .../scripts/cc_python_parser/function_data.py | 4 +-- .../persistence/import_dto.py | 13 +++++----- .../persistence/persistence.py | 9 ++++++- .../scripts/cc_python_parser/scope_manager.py | 8 +++--- .../cc_python_parser/symbol_collector.py | 26 +++++++++++++------ .../scripts/cc_python_parser/variable_data.py | 8 +++--- 8 files changed, 47 insertions(+), 29 deletions(-) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/base_data.py b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py index 26051dc14..fb04a6af3 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/base_data.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py @@ -57,8 +57,8 @@ def __init__(self, name: str, qualified_name: str, pos: FilePosition): class ImportedDeclaration(Generic[T]): - def __init__(self, declaration: T, module, pos: FilePosition): - self.imported_declaration = declaration + def __init__(self, qualified_name: str, declaration: T, module, pos: FilePosition): + self.qualified_name = qualified_name self.module = module self.position = pos diff --git a/plugins/python/parser/src/scripts/cc_python_parser/class_data.py b/plugins/python/parser/src/scripts/cc_python_parser/class_data.py index 2005197de..39d3dda09 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/class_data.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/class_data.py @@ -55,7 +55,7 @@ def get_type_repr(self) -> str: class ImportedClassDeclaration(ClassDeclaration, ImportedDeclaration[ClassDeclaration]): - def __init__(self, name: str, pos: FilePosition, class_declaration: ClassDeclaration, module): + def __init__(self, qualified_name: str, name: str, pos: FilePosition, class_declaration: ClassDeclaration, module): ClassDeclaration.__init__(self, name, "", pos, class_declaration.base_classes, "") - ImportedDeclaration.__init__(self, class_declaration, module, pos) + ImportedDeclaration.__init__(self, qualified_name, class_declaration, module, pos) self.type = {class_declaration} diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_data.py b/plugins/python/parser/src/scripts/cc_python_parser/function_data.py index 9beaf4c51..f8fdde439 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/function_data.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_data.py @@ -47,6 +47,6 @@ class StaticFunctionDeclaration(FunctionDeclaration): class ImportedFunctionDeclaration(FunctionDeclaration, data.ImportedDeclaration[FunctionDeclaration]): - def __init__(self, name: str, pos: FilePosition, func_declaration: FunctionDeclaration, module): + def __init__(self, qualified_name: str, name: str, pos: FilePosition, func_declaration: FunctionDeclaration, module): FunctionDeclaration.__init__(self, name, "", pos, func_declaration.parameters, "", func_declaration.type) - data.ImportedDeclaration.__init__(self, func_declaration, module, pos) + data.ImportedDeclaration.__init__(self, qualified_name, func_declaration, module, pos) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py index 5dcf51444..181110eaa 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/import_dto.py @@ -1,5 +1,5 @@ from pathlib import PurePath -from typing import Set, Dict, List +from typing import Dict, List from cc_python_parser.common.file_position import FilePosition from cc_python_parser.persistence.base_dto import create_file_position_dto @@ -7,7 +7,8 @@ class ImportDataDTO: - def __init__(self, imported: PurePath, pos: FilePosition): + def __init__(self, qualified_name: str, imported: PurePath, pos: FilePosition): + self.qualified_name: str = qualified_name self.imported: str = str(imported) self.position: FilePositionDTO = create_file_position_dto(pos) @@ -27,11 +28,11 @@ def __init__(self, importer: PurePath): self.imported_modules: List[ImportDataDTO] = [] self.imported_symbols: Dict[ImportDataDTO, List[str]] = {} - def add_module_import(self, imported: PurePath, pos: FilePosition): - self.imported_modules.append(ImportDataDTO(imported, pos)) + def add_module_import(self, qualified_name: str, imported: PurePath, pos: FilePosition): + self.imported_modules.append(ImportDataDTO(qualified_name, imported, pos)) - def add_symbol_import(self, imported: PurePath, pos: FilePosition, symbol_qualified_name: str): - imported_data_dto = ImportDataDTO(imported, pos) + def add_symbol_import(self, qualified_name: str, imported: PurePath, pos: FilePosition, symbol_qualified_name: str): + imported_data_dto = ImportDataDTO(qualified_name, imported, pos) if imported_data_dto not in self.imported_symbols: self.imported_symbols[imported_data_dto] = [] self.imported_symbols[imported_data_dto].append(symbol_qualified_name) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py index 8dfd63ee6..298e670db 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py @@ -64,9 +64,16 @@ def persist_class(self, declaration: ClassDeclarationDTO) -> None: def persist_import(self, imports: ImportDTO) -> None: if self.c_persistence is not None: self.c_persistence.print("Persist import") + for i in imports.imported_modules: + self.c_persistence.print(f"Persist imported module: {i.qualified_name}") + for i in imports.imported_symbols: + self.c_persistence.print(f"Persist imported symbol: {i.qualified_name}") self.c_persistence.persist_import(imports) else: - self.log.write("Persist import\n") + for i in imports.imported_modules: + self.log.write(f"Persist imported module: {i.qualified_name}\n") + for i in imports.imported_symbols: + self.log.write(f"Persist imported symbol: {i.qualified_name}\n") model_persistence = ModelPersistence(None) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py index 0f4e5f083..041445bd7 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py @@ -449,23 +449,23 @@ def persist_current_scope(self): for var in self.scopes[-1].variable_declarations: if isinstance(var, ImportedDeclaration): - import_dto.add_symbol_import(var.module.location, var.position, + import_dto.add_symbol_import(var.qualified_name, var.module.location, var.position, var.imported_declaration.qualified_name) elif isinstance(var, ModuleVariableDeclaration): - import_dto.add_module_import(var.imported_module_location, var.file_position) + import_dto.add_module_import(var.qualified_name, var.imported_module_location, var.file_position) else: self.persistence.persist_variable(var.create_dto()) for func in self.scopes[-1].function_declarations: if isinstance(func, ImportedDeclaration): - import_dto.add_symbol_import(func.module.location, func.position, + import_dto.add_symbol_import(func.qualified_name, func.module.location, func.position, func.imported_declaration.qualified_name) else: self.persistence.persist_function(func.create_dto()) for cl in self.scopes[-1].class_declarations: if isinstance(cl, ImportedDeclaration): - import_dto.add_symbol_import(cl.module.location, cl.position, + import_dto.add_symbol_import(cl.qualified_name, cl.module.location, cl.position, cl.imported_declaration.qualified_name) else: self.persistence.persist_class(cl.create_dto()) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py index f7d3d40b4..736092267 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py @@ -66,8 +66,11 @@ def visit_Import(self, node: ast.Import) -> Any: if not module.is_module_import(): continue # scope.imports.append((module, self.import_finder.get_global_scope_by_location(module.location))) + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(module.module.name, + r.start_position.line) self.scope_manager.append_module_variable_to_current_scope( - ModuleVariableDeclaration(module.path[-1], module.location, + ModuleVariableDeclaration(qualified_name, module.path[-1], module.location, FilePosition(self.current_file, module.range), self.import_finder.get_global_scope_by_location(module.location))) self.scope_manager.append_import(node) @@ -77,8 +80,11 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: metrics.add_import_count() modules = ImportTable.convert_ast_import_from_to_import(node) for module in modules: + r = create_range_from_ast_node(node) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(module.module.name, + r.start_position.line) self.scope_manager.append_module_variable_to_current_scope( - ModuleVariableDeclaration(module.path[-1], module.location, + ModuleVariableDeclaration(qualified_name, module.path[-1], module.location, FilePosition(self.current_file, module.range), self.import_finder.get_global_scope_by_location(module.location))) if not module.is_module_import(): @@ -92,18 +98,20 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: for imported in module.imported: if imported.is_all_imported(): for declaration in scope_manager.get_global_scope().declarations: + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(declaration.name, + r.start_position.line) if isinstance(declaration, VariableDeclaration): - var = ImportedVariableDeclaration( + var = ImportedVariableDeclaration(qualified_name, declaration.name, file_position, declaration, module) self.scope_manager.append_variable_to_current_scope(var) self.imported_declaration_scope_map[var] = scope_manager elif isinstance(declaration, FunctionDeclaration): - func = ImportedFunctionDeclaration( + func = ImportedFunctionDeclaration(qualified_name, declaration.name, file_position, declaration, module) self.scope_manager.append_function_to_current_scope(func) self.imported_declaration_scope_map[func] = scope_manager elif isinstance(declaration, ClassDeclaration): - c = ImportedClassDeclaration( + c = ImportedClassDeclaration(qualified_name, declaration.name, file_position, declaration, module) self.scope_manager.append_class_to_current_scope(c) self.imported_declaration_scope_map[c] = scope_manager @@ -112,22 +120,24 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> Any: else: # TODO: redefinition? declaration = scope_manager.get_declaration(imported.name) + qualified_name = self.scope_manager.get_qualified_name_from_current_scope(imported.name, + r.start_position.line) if isinstance(declaration, PlaceholderType): continue elif isinstance(declaration, VariableDeclaration): - var = ImportedVariableDeclaration( + var = ImportedVariableDeclaration(qualified_name, imported.name if imported.alias is None else imported.alias, file_position, declaration, module) self.scope_manager.append_variable_to_current_scope(var) self.imported_declaration_scope_map[var] = scope_manager elif isinstance(declaration, FunctionDeclaration): - func = ImportedFunctionDeclaration( + func = ImportedFunctionDeclaration(qualified_name, imported.name if imported.alias is None else imported.alias, file_position, declaration, module) self.scope_manager.append_function_to_current_scope(func) self.imported_declaration_scope_map[func] = scope_manager elif isinstance(declaration, ClassDeclaration): - c = ImportedClassDeclaration( + c = ImportedClassDeclaration(qualified_name, imported.name if imported.alias is None else imported.alias, file_position, declaration, module) self.scope_manager.append_class_to_current_scope(c) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py b/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py index ee9ad52bb..5764f088c 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/variable_data.py @@ -63,16 +63,16 @@ def __init__(self, name: str, qualified_name: str, pos: FilePosition, reference: # TODO: short/long name? (in case of import) class ModuleVariableDeclaration(VariableDeclaration): # module: Union[GlobalScope, PreprocessedFile] - circular import - def __init__(self, name: str, location: PurePath, pos: FilePosition, module): - super().__init__(name, "", pos, {Module()}) # TODO: need qualified name? + def __init__(self, qualified_name: str, name: str, location: PurePath, pos: FilePosition, module): + super().__init__(name, qualified_name, pos, {Module()}) self.imported_module = module self.imported_module_location: PurePath = location class ImportedVariableDeclaration(VariableDeclaration, ImportedDeclaration[VariableDeclaration]): - def __init__(self, name: str, pos: FilePosition, var_declaration: VariableDeclaration, module): + def __init__(self, qualified_name: str, name: str, pos: FilePosition, var_declaration: VariableDeclaration, module): VariableDeclaration.__init__(self, name, "", pos, var_declaration.type) - ImportedDeclaration.__init__(self, var_declaration, module, pos) + ImportedDeclaration.__init__(self, qualified_name, var_declaration, module, pos) class TypeVariableDeclaration(ReferenceVariableDeclaration): From e7b2db16c2c8c53cf308df5497107917fcb70b74 Mon Sep 17 00:00:00 2001 From: Bence Date: Sat, 27 Nov 2021 14:36:26 +0100 Subject: [PATCH 34/39] Handle qualified name in ImportData on c++ side. --- plugins/python/parser/src/pythonparser.cpp | 246 ++++++++------------- 1 file changed, 97 insertions(+), 149 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index 3110005b2..fcdf6e923 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -68,21 +67,19 @@ class Persistence model::PythonEntityPtr getPythonEntity(const std::string& qualifiedName); model::PythonClassPtr getPythonClass(const std::string& qualifiedName); model::PythonVariablePtr getPythonVariable(const std::string& qualifiedName); - bool isAstNodePersisted(const model::PythonAstNodePtr& node) const; - bool isAstNodePersisted(const std::set& nodes, const model::PythonAstNodePtr& node) const; - - std::set _astNodes; - std::set _variables; - std::map> _variableUsages; - std::set _functions; - std::map> _functionUsages; - std::set _classes; - std::map> _classUsages; - std::set _members; - std::set _inheritance; - std::set _imports; - std::set _documentations; - std::set _types; + + std::vector _astNodes; + std::vector _variables; + std::map> _variableUsages; + std::vector _functions; + std::map> _functionUsages; + std::vector _classes; + std::map> _classUsages; + std::vector _members; + std::vector _inheritance; + std::vector _imports; + std::vector _documentations; + std::vector _types; }; template @@ -99,36 +96,24 @@ Persistence::~Persistence() ctx.srcMgr.persistFiles(); (util::OdbTransaction(ctx.db))([this]{ - std::vector tmp(_astNodes.begin(), _astNodes.end()); - util::persistAll(tmp, ctx.db); - for(auto& ast : _variableUsages){ - std::vector v(ast.second.begin(), ast.second.end()); - util::persistAll(v, ctx.db); - } - for(auto& ast : _functionUsages){ - std::vector v(ast.second.begin(), ast.second.end()); - util::persistAll(v, ctx.db); - } - for(auto& ast : _classUsages){ - std::vector v(ast.second.begin(), ast.second.end()); - util::persistAll(v, ctx.db); - } - std::vector tmp_variables(_variables.begin(), _variables.end()); - util::persistAll(tmp_variables, ctx.db); - std::vector tmp_functions(_functions.begin(), _functions.end()); - util::persistAll(tmp_functions, ctx.db); - std::vector tmp_classes(_classes.begin(), _classes.end()); - util::persistAll(tmp_classes, ctx.db); - std::vector tmp_members(_members.begin(), _members.end()); - util::persistAll(tmp_members, ctx.db); - std::vector tmp_inheritance(_inheritance.begin(), _inheritance.end()); - util::persistAll(tmp_inheritance, ctx.db); - std::vector tmp_imports(_imports.begin(), _imports.end()); - util::persistAll(tmp_imports, ctx.db); - std::vector tmp_documentations(_documentations.begin(), _documentations.end()); - util::persistAll(tmp_documentations, ctx.db); - std::vector tmp_types(_types.begin(), _types.end()); - util::persistAll(tmp_types, ctx.db); + util::persistAll(_astNodes, ctx.db); + for(auto& ast : _variableUsages){ + util::persistAll(ast.second, ctx.db); + } + for(auto& ast : _functionUsages){ + util::persistAll(ast.second, ctx.db); + } + for(auto& ast : _classUsages){ + util::persistAll(ast.second, ctx.db); + } + util::persistAll(_variables, ctx.db); + util::persistAll(_functions, ctx.db); + util::persistAll(_classes, ctx.db); + util::persistAll(_members, ctx.db); + util::persistAll(_inheritance, ctx.db); + util::persistAll(_imports, ctx.db); + util::persistAll(_documentations, ctx.db); + util::persistAll(_types, ctx.db); }); } @@ -220,11 +205,11 @@ void Persistence::persistVariable(boost::python::object pyVariable) varAstNode->id = model::createIdentifier(*varAstNode); - if(isAstNodePersisted(varAstNode)){ + if(std::find(_astNodes.begin(), _astNodes.end(), varAstNode) == _astNodes.end()){ return; } - _astNodes.insert(varAstNode); + _astNodes.push_back(varAstNode); model::PythonVariablePtr variable(new model::PythonVariable); variable->astNodeId = varAstNode->id; variable->name = boost::python::extract(name); @@ -232,7 +217,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) variable->visibility = boost::python::extract(visibility); variable->id = model::createIdentifier(*variable); - _variables.insert(variable); + _variables.push_back(variable); for(int i = 0; itype = t->id; type->symbol = variable->id; - _types.insert(type); + _types.push_back(type); } _variableUsages[variable->id] = {}; @@ -261,11 +246,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_variableUsages[variable->id], usageAstNode)) { - continue; - } - - _variableUsages[variable->id].insert(usageAstNode); + _variableUsages[variable->id].push_back(usageAstNode); } } catch (const odb::object_already_persistent& ex) @@ -311,11 +292,11 @@ void Persistence::persistFunction(boost::python::object pyFunction) funcAstNode->id = model::createIdentifier(*funcAstNode); - if(isAstNodePersisted(funcAstNode)){ + if(std::find(_astNodes.begin(), _astNodes.end(), funcAstNode) == _astNodes.end()){ return; } - _astNodes.insert(funcAstNode); + _astNodes.push_back(funcAstNode); model::PythonFunctionPtr function(new model::PythonFunction); function->astNodeId = funcAstNode->id; @@ -341,14 +322,14 @@ void Persistence::persistFunction(boost::python::object pyFunction) function->locals.push_back(local); } - _functions.insert(function); + _functions.push_back(function); model::PythonDocumentationPtr documentation(new model::PythonDocumentation); documentation->documentation = boost::python::extract(pyDocumentation); documentation->documented = function->id; documentation->documentationKind = model::PythonDocumentation::Function; - _documentations.insert(documentation); + _documentations.push_back(documentation); for(int i = 0; itype = t->id; type->symbol = function->id; - _types.insert(type); + _types.push_back(type); } _functionUsages[function->id] = {}; @@ -377,11 +358,7 @@ void Persistence::persistFunction(boost::python::object pyFunction) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_functionUsages[function->id], usageAstNode)) { - continue; - } - - _functionUsages[function->id].insert(usageAstNode); + _functionUsages[function->id].push_back(usageAstNode); } } catch(std::exception e){ std::cout << "Func exception:" << e.what() << std::endl; @@ -409,11 +386,11 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) classAstNode->id = model::createIdentifier(*classAstNode); - if(isAstNodePersisted(classAstNode)){ + if(std::find(_astNodes.begin(), _astNodes.end(), classAstNode) == _astNodes.end()){ return; } - _astNodes.insert(classAstNode); + _astNodes.push_back(classAstNode); model::PythonClassPtr cl(new model::PythonClass); cl->astNodeId = classAstNode->id; @@ -423,7 +400,7 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) cl->id = model::createIdentifier(*cl); - _classes.insert(cl); + _classes.push_back(cl); } catch(std::exception e){ std::cout << "Preprocessed class exception:" << e.what() << std::endl; @@ -459,7 +436,8 @@ void Persistence::persistClass(boost::python::object pyClass) documentation->documented = cl->id; documentation->documentationKind = model::PythonDocumentation::Class; - _documentations.insert(documentation); + _documentations.push_back(documentation); + _classUsages[cl->id] = {}; for(int i = 0; i usageFileLoc = @@ -475,24 +453,17 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_classUsages[cl->id], usageAstNode)) { - continue; - } - - _classUsages[cl->id].insert(usageAstNode); + _classUsages[cl->id].push_back(usageAstNode); } for(int i = 0; iderived = cl->id; std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); - auto baseClass = getPythonEntity(baseClassQualifiedName); - if(baseClass == nullptr){ - continue; - } - inheritance->base = baseClass->id; - _inheritance.insert(inheritance); + inheritance->base = getPythonEntity(baseClassQualifiedName)->id; + + _inheritance.push_back(inheritance); } boost::python::list methods = boost::python::extract(members.attr("methods")); @@ -512,7 +483,6 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ std::cout << "method is none" << std::endl; - continue; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -520,15 +490,15 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Method; classMember->staticMember = false; - _members.insert(classMember); + _members.push_back(classMember); boost::python::list _usages = boost::python::extract(methods[i].attr("usages")); - std::set& _funcUsages = _functionUsages[method->id]; + std::vector& _funcUsages = _functionUsages[method->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || - std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _funcUsages.end()) + std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _funcUsages.end()) { continue; } @@ -540,11 +510,7 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_funcUsages, usageAstNode)) { - continue; - } - - _funcUsages.insert(usageAstNode); + _funcUsages.push_back(usageAstNode); } } @@ -554,7 +520,6 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ std::cout << "static method is none" << std::endl; - continue; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -562,10 +527,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Method; classMember->staticMember = true; - _members.insert(classMember); + _members.push_back(classMember); boost::python::list _usages = boost::python::extract(staticMethods[i].attr("usages")); - std::set& _funcUsages = _functionUsages[method->id]; + std::vector& _funcUsages = _functionUsages[method->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -582,11 +547,7 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_funcUsages, usageAstNode)) { - continue; - } - - _funcUsages.insert(usageAstNode); + _funcUsages.push_back(usageAstNode); } } @@ -599,7 +560,6 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ std::cout << "attr is none" << std::endl; - continue; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -607,10 +567,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = false; - _members.insert(classMember); + _members.push_back(classMember); boost::python::list _usages = boost::python::extract(attributes[i].attr("usages")); - std::set& _varUsages = _variableUsages[attr->id]; + std::vector& _varUsages = _variableUsages[attr->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -627,11 +587,7 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_varUsages, usageAstNode)) { - continue; - } - - _varUsages.insert(usageAstNode); + _varUsages.push_back(usageAstNode); } } @@ -641,7 +597,6 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ std::cout << "static attr is none" << std::endl; - continue; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -649,10 +604,10 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Attribute; classMember->staticMember = true; - _members.insert(classMember); + _members.push_back(classMember); boost::python::list _usages = boost::python::extract(staticAttributes[i].attr("usages")); - std::set& _varUsages = _variableUsages[attr->id]; + std::vector& _varUsages = _variableUsages[attr->id]; for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -669,11 +624,7 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_varUsages, usageAstNode)) { - continue; - } - - _varUsages.insert(usageAstNode); + _varUsages.push_back(usageAstNode); } } @@ -683,7 +634,6 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr inner = getPythonEntity(qualifiedName); if(inner == nullptr){ std::cout << "inner class is none" << std::endl; - continue; } classMember->astNodeId = inner->astNodeId; classMember->memberId = inner->id; @@ -691,7 +641,28 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->kind = model::PythonClassMember::Class; classMember->staticMember = false; - _members.insert(classMember); + _members.push_back(classMember); + + boost::python::list _usages = boost::python::extract(classes[i].attr("usages")); + std::vector& _clUsages = _classUsages[cl->id]; + for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_clUsages.begin(), _clUsages.end(), [&](const auto& p){ + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _clUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + _clUsages.push_back(usageAstNode); + } } } catch(std::exception e){ std::cout << "Class exception:" << e.what() << std::endl; @@ -718,6 +689,7 @@ void Persistence::persistImport(boost::python::object pyImport) model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); boost::optional fileLoc = createFileLocFromPythonFilePosition(importData.attr("position")); + std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); if (moduleFile == nullptr || fileLoc == boost::none) { continue; @@ -725,13 +697,13 @@ void Persistence::persistImport(boost::python::object pyImport) model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); moduleAstNode->location = fileLoc.get(); - moduleAstNode->qualifiedName = ""; + moduleAstNode->qualifiedName = qualifiedName; moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; moduleAstNode->id = model::createIdentifier(*moduleAstNode); - if(isAstNodePersisted(moduleAstNode)){ + if(std::find(_astNodes.begin(), _astNodes.end(), moduleAstNode) == _astNodes.end()){ continue; } @@ -740,8 +712,8 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->importer = file; moduleImport->imported = moduleFile; - _astNodes.insert(moduleAstNode); - _imports.insert(moduleImport); + _astNodes.push_back(moduleAstNode); + _imports.push_back(moduleImport); } boost::python::list importDict = importedSymbols.items(); @@ -752,6 +724,7 @@ void Persistence::persistImport(boost::python::object pyImport) model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); boost::optional fileLoc = createFileLocFromPythonFilePosition(importData.attr("position")); + std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); if (moduleFile == nullptr || fileLoc == boost::none) { continue; @@ -759,13 +732,13 @@ void Persistence::persistImport(boost::python::object pyImport) model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); moduleAstNode->location = fileLoc.get(); - moduleAstNode->qualifiedName = ""; + moduleAstNode->qualifiedName = qualifiedName; moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; moduleAstNode->id = model::createIdentifier(*moduleAstNode); - if(isAstNodePersisted(moduleAstNode)){ + if(std::find(_astNodes.begin(), _astNodes.end(), moduleAstNode) == _astNodes.end()){ continue; } @@ -775,15 +748,12 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->importer = file; moduleImport->imported = moduleFile; auto symb = getPythonEntity(boost::python::extract(import[1][j])); - if(symb == nullptr){ - continue; - } moduleImport->importedSymbol = symb->id; - _imports.insert(moduleImport); + _imports.push_back(moduleImport); } - _astNodes.insert(moduleAstNode); + _astNodes.push_back(moduleAstNode); } } catch(std::exception e){ std::cout << "Import exception:" << e.what() << std::endl; @@ -872,28 +842,6 @@ model::PythonClassPtr Persistence::getPythonClass(const std::string& qualifiedNa return nullptr; } -bool Persistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const -{ - for(auto it : _astNodes){ - if(*it == *node){ - return true; - } - } - - return false; -} - -bool Persistence::isAstNodePersisted(const std::set& nodes, const model::PythonAstNodePtr& node) const -{ - for(auto it : nodes){ - if(*it == *node){ - return true; - } - } - - return false; -} - typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ From d6cc131428d5ea0c7b9ce87e11dec19573fbb024 Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Mon, 29 Nov 2021 10:15:00 +0100 Subject: [PATCH 35/39] Check PythonAstNode if already exists during persisting. Remove unnecessary inner class usage persisting. Add missing imported declaration to ImportedDeclaration --- plugins/python/parser/src/pythonparser.cpp | 92 +++++++++++++------ .../src/scripts/cc_python_parser/base_data.py | 1 + 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index fcdf6e923..b23e5adda 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -67,6 +67,8 @@ class Persistence model::PythonEntityPtr getPythonEntity(const std::string& qualifiedName); model::PythonClassPtr getPythonClass(const std::string& qualifiedName); model::PythonVariablePtr getPythonVariable(const std::string& qualifiedName); + bool isAstNodePersisted(const model::PythonAstNodePtr& node) const; + bool isAstNodePersisted(const std::vector& nodes, const model::PythonAstNodePtr& node) const; std::vector _astNodes; std::vector _variables; @@ -205,7 +207,7 @@ void Persistence::persistVariable(boost::python::object pyVariable) varAstNode->id = model::createIdentifier(*varAstNode); - if(std::find(_astNodes.begin(), _astNodes.end(), varAstNode) == _astNodes.end()){ + if(isAstNodePersisted(varAstNode)){ return; } @@ -246,6 +248,10 @@ void Persistence::persistVariable(boost::python::object pyVariable) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_variableUsages[variable->id], usageAstNode)) { + continue; + } + _variableUsages[variable->id].push_back(usageAstNode); } @@ -292,7 +298,7 @@ void Persistence::persistFunction(boost::python::object pyFunction) funcAstNode->id = model::createIdentifier(*funcAstNode); - if(std::find(_astNodes.begin(), _astNodes.end(), funcAstNode) == _astNodes.end()){ + if(isAstNodePersisted(funcAstNode)){ return; } @@ -358,6 +364,10 @@ void Persistence::persistFunction(boost::python::object pyFunction) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_functionUsages[function->id], usageAstNode)) { + continue; + } + _functionUsages[function->id].push_back(usageAstNode); } } catch(std::exception e){ @@ -386,7 +396,7 @@ void Persistence::persistPreprocessedClass(boost::python::object pyClass) classAstNode->id = model::createIdentifier(*classAstNode); - if(std::find(_astNodes.begin(), _astNodes.end(), classAstNode) == _astNodes.end()){ + if(isAstNodePersisted(classAstNode)){ return; } @@ -429,6 +439,7 @@ void Persistence::persistClass(boost::python::object pyClass) if (cl == nullptr){ std::cout << "cl is none" << std::endl; + return; } model::PythonDocumentationPtr documentation(new model::PythonDocumentation); @@ -453,6 +464,10 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_classUsages[cl->id], usageAstNode)) { + continue; + } + _classUsages[cl->id].push_back(usageAstNode); } @@ -483,6 +498,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ std::cout << "method is none" << std::endl; + continue; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -510,6 +526,10 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_funcUsages, usageAstNode)) { + continue; + } + _funcUsages.push_back(usageAstNode); } } @@ -520,6 +540,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(method == nullptr){ std::cout << "static method is none" << std::endl; + continue; } classMember->astNodeId = method->astNodeId; classMember->memberId = method->id; @@ -547,6 +568,10 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_funcUsages, usageAstNode)) { + continue; + } + _funcUsages.push_back(usageAstNode); } } @@ -560,6 +585,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ std::cout << "attr is none" << std::endl; + continue; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -587,6 +613,10 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_varUsages, usageAstNode)) { + continue; + } + _varUsages.push_back(usageAstNode); } } @@ -597,6 +627,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(attr == nullptr){ std::cout << "static attr is none" << std::endl; + continue; } classMember->astNodeId = attr->astNodeId; classMember->memberId = attr->id; @@ -624,6 +655,10 @@ void Persistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); + if(isAstNodePersisted(_varUsages, usageAstNode)) { + continue; + } + _varUsages.push_back(usageAstNode); } } @@ -634,6 +669,7 @@ void Persistence::persistClass(boost::python::object pyClass) model::PythonEntityPtr inner = getPythonEntity(qualifiedName); if(inner == nullptr){ std::cout << "inner class is none" << std::endl; + continue; } classMember->astNodeId = inner->astNodeId; classMember->memberId = inner->id; @@ -642,27 +678,6 @@ void Persistence::persistClass(boost::python::object pyClass) classMember->staticMember = false; _members.push_back(classMember); - - boost::python::list _usages = boost::python::extract(classes[i].attr("usages")); - std::vector& _clUsages = _classUsages[cl->id]; - for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); - if(_fl == boost::none || - std::find_if(_clUsages.begin(), _clUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _clUsages.end()) - { - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = _fl.get(); - usageAstNode->qualifiedName = qualifiedName; - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - _clUsages.push_back(usageAstNode); - } } } catch(std::exception e){ std::cout << "Class exception:" << e.what() << std::endl; @@ -703,7 +718,7 @@ void Persistence::persistImport(boost::python::object pyImport) moduleAstNode->id = model::createIdentifier(*moduleAstNode); - if(std::find(_astNodes.begin(), _astNodes.end(), moduleAstNode) == _astNodes.end()){ + if(isAstNodePersisted(moduleAstNode)){ continue; } @@ -738,7 +753,7 @@ void Persistence::persistImport(boost::python::object pyImport) moduleAstNode->id = model::createIdentifier(*moduleAstNode); - if(std::find(_astNodes.begin(), _astNodes.end(), moduleAstNode) == _astNodes.end()){ + if(isAstNodePersisted(moduleAstNode)){ continue; } @@ -748,6 +763,9 @@ void Persistence::persistImport(boost::python::object pyImport) moduleImport->importer = file; moduleImport->imported = moduleFile; auto symb = getPythonEntity(boost::python::extract(import[1][j])); + if(symb == nullptr){ + continue; + } moduleImport->importedSymbol = symb->id; _imports.push_back(moduleImport); @@ -842,6 +860,28 @@ model::PythonClassPtr Persistence::getPythonClass(const std::string& qualifiedNa return nullptr; } +bool Persistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const +{ + for(auto it : _astNodes){ + if(*it == *node){ + return true; + } + } + + return false; +} + +bool Persistence::isAstNodePersisted(const std::vector& nodes, const model::PythonAstNodePtr& node) const +{ + for(auto it : nodes){ + if(*it == *node){ + return true; + } + } + + return false; +} + typedef boost::shared_ptr PersistencePtr; BOOST_PYTHON_MODULE(persistence){ diff --git a/plugins/python/parser/src/scripts/cc_python_parser/base_data.py b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py index fb04a6af3..da664bd0b 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/base_data.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/base_data.py @@ -59,6 +59,7 @@ def __init__(self, name: str, qualified_name: str, pos: FilePosition): class ImportedDeclaration(Generic[T]): def __init__(self, qualified_name: str, declaration: T, module, pos: FilePosition): self.qualified_name = qualified_name + self.imported_declaration = declaration self.module = module self.position = pos From 2856131aa023ab12d92168d0679282cdc40c5760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Cser=C3=A9p?= Date: Wed, 23 Feb 2022 11:51:45 +0100 Subject: [PATCH 36/39] Update GitLab CI from upstream --- .github/workflows/ci.yml | 63 ++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b588b5a9..505b9d4f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,11 @@ jobs: run: > sudo apt-get install -y git cmake make g++ gcc-7-plugin-dev libboost-all-dev llvm-10-dev clang-10 libclang-10-dev default-jdk libssl1.0-dev libgraphviz-dev - libmagic-dev libgit2-dev ctags libgtest-dev npm libldap2-dev + libmagic-dev libgit2-dev ctags doxygen libgtest-dev npm libldap2-dev + + - name: Remove default Postgresql Ubuntu 18 + if: ${{ matrix.os == 'ubuntu-18.04' && matrix.db == 'postgresql' }} + run: sudo apt-get remove libpq5 - name: Install Postgresql Ubuntu 18 if: ${{ matrix.os == 'ubuntu-18.04' && matrix.db == 'postgresql' }} @@ -127,7 +131,7 @@ jobs: run: > sudo apt-get install -y git cmake make g++ libboost-all-dev llvm-10-dev clang-10 libclang-10-dev odb libodb-dev thrift-compiler libthrift-dev default-jdk libssl-dev - libgraphviz-dev libmagic-dev libgit2-dev ctags libgtest-dev npm libldap2-dev + libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen libgtest-dev npm libldap2-dev - name: Install Postgresql Ubuntu 20 if: ${{ matrix.os == 'ubuntu-20.04' && matrix.db == 'postgresql' }} @@ -258,6 +262,10 @@ jobs: llvm-10-dev clang-10 libclang-10-dev default-jre libssl1.0-dev libmagic-dev libgit2-dev ctags libgtest-dev libldap-2.4-2 + - name: Remove default Postgresql Ubuntu 18 + if: ${{ matrix.os == 'ubuntu-18.04' && matrix.db == 'postgresql' }} + run: sudo apt-get remove libpq5 + - name: Install Postgresql Ubuntu 18 if: ${{ matrix.os == 'ubuntu-18.04' && matrix.db == 'postgresql' }} run: sudo apt-get install postgresql-server-dev-10 @@ -393,22 +401,34 @@ jobs: docker: needs: parse - if: ${{ github.repository == 'Ericsson/CodeCompass' && github.ref == 'refs/heads/master' }} + if: ${{ github.repository == 'Ericsson/CodeCompass' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release/') == true) }} runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 + - name: Branch name slug + uses: rlespinasse/github-slug-action@v4 + + - name: Branch name substring + uses: bhowell2/github-substring-action@v1.0.0 + id: branch_substring + with: + value: ${{ env.GITHUB_REF_SLUG }} + index_of_str: "release-" + fail_if_not_found: false + default_return_value: "" + - name: Build images - id: docker_build run: | - docker build -t codecompass:dev -t modelcpp/codecompass:dev --file docker/dev/Dockerfile . - docker build -t codecompass:runtime -t modelcpp/codecompass:runtime-sqlite --file docker/runtime/Dockerfile --no-cache --build-arg CC_DATABASE=sqlite . - docker build -t codecompass:web -t modelcpp/codecompass:web-sqlite --file docker/web/Dockerfile --no-cache . - docker build -t codecompass:runtime -t modelcpp/codecompass:runtime-pgsql --file docker/runtime/Dockerfile --no-cache --build-arg CC_DATABASE=pgsql . - docker build -t codecompass:web -t modelcpp/codecompass:web-pgsql --file docker/web/Dockerfile --no-cache . - docker tag modelcpp/codecompass:runtime-pgsql modelcpp/codecompass:latest + BRANCH_PREFIX=${{ steps.branch_substring.outputs.substring }} + BRANCH_PREFIX=${BRANCH_PREFIX}${BRANCH_PREFIX:+-} # append dash if not empty + docker build -t codecompass:dev -t modelcpp/codecompass:${BRANCH_PREFIX}dev --file docker/dev/Dockerfile . + docker build -t codecompass:runtime -t modelcpp/codecompass:${BRANCH_PREFIX}runtime-sqlite --file docker/runtime/Dockerfile --no-cache --build-arg CC_DATABASE=sqlite . + docker build -t codecompass:web -t modelcpp/codecompass:${BRANCH_PREFIX}web-sqlite --file docker/web/Dockerfile --no-cache . + docker build -t codecompass:runtime -t modelcpp/codecompass:${BRANCH_PREFIX}runtime-pgsql --file docker/runtime/Dockerfile --no-cache --build-arg CC_DATABASE=pgsql . + docker build -t codecompass:web -t modelcpp/codecompass:${BRANCH_PREFIX}web-pgsql --file docker/web/Dockerfile --no-cache . - name: Login to DockerHub uses: docker/login-action@v1 @@ -418,16 +438,23 @@ jobs: - name: Push images run: | - docker push modelcpp/codecompass:dev - docker push modelcpp/codecompass:runtime-sqlite - docker push modelcpp/codecompass:runtime-pgsql - docker push modelcpp/codecompass:web-sqlite - docker push modelcpp/codecompass:web-pgsql + BRANCH_PREFIX=${{ steps.branch_substring.outputs.substring }} + BRANCH_PREFIX=${BRANCH_PREFIX}${BRANCH_PREFIX:+-} # append dash if not empty + docker push modelcpp/codecompass:${BRANCH_PREFIX}dev + docker push modelcpp/codecompass:${BRANCH_PREFIX}runtime-sqlite + docker push modelcpp/codecompass:${BRANCH_PREFIX}runtime-pgsql + docker push modelcpp/codecompass:${BRANCH_PREFIX}web-sqlite + docker push modelcpp/codecompass:${BRANCH_PREFIX}web-pgsql + + - name: Tag and push latest image + if: ${{ github.ref_name == 'master' }} + run: | + docker tag modelcpp/codecompass:runtime-pgsql modelcpp/codecompass:latest docker push modelcpp/codecompass:latest tarball: needs: parse - if: ${{ github.repository == 'Ericsson/CodeCompass' && github.ref == 'refs/heads/master' }} + if: ${{ github.repository == 'Ericsson/CodeCompass' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release/') == true) }} runs-on: ubuntu-20.04 steps: @@ -437,5 +464,5 @@ jobs: - name: Install curl run: sudo apt-get install curl - - name: Trigget GitLab CI - run: curl -X POST -F token=${{ secrets.GITLAB_TRIGGER_TOKEN }} -F ref=master https://gitlab.inf.elte.hu/api/v4/projects/85/trigger/pipeline + - name: Trigger GitLab CI + run: curl -X POST -F token=${{ secrets.GITLAB_TRIGGER_TOKEN }} -F ref=${{ github.ref_name }} https://gitlab.inf.elte.hu/api/v4/projects/85/trigger/pipeline \ No newline at end of file From b70b911aac9c28f127a8ef9490d969a8178d4dbe Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sat, 23 Apr 2022 20:47:34 +0200 Subject: [PATCH 37/39] Use logutil Introduce skip file Small refactors --- plugins/python/parser/CMakeLists.txt | 7 +- .../include/pythonparser/pythonpersistence.h | 105 ++ plugins/python/parser/src/pythonparser.cpp | 925 +--------------- .../python/parser/src/pythonpersistence.cpp | 993 ++++++++++++++++++ .../cc_python_parser/common/__init__.py | 2 +- .../cc_python_parser/common/hashable_list.py | 12 - .../cc_python_parser/common/unique_list.py | 23 - .../scripts/cc_python_parser/common/utils.py | 7 +- .../src/scripts/cc_python_parser/file_info.py | 9 +- .../function_symbol_collector.py | 8 +- .../cc_python_parser/import_preprocessor.py | 5 +- .../member_access_collector.py | 7 +- .../src/scripts/cc_python_parser/parser.py | 24 +- .../persistence/persistence.py | 44 +- .../scripts/cc_python_parser/python_parser.py | 19 +- .../scripts/cc_python_parser/scope_manager.py | 15 +- .../src/scripts/cc_python_parser/skip_file.py | 46 + .../cc_python_parser/symbol_collector.py | 20 +- .../cc_python_parser/type_deduction.py | 9 +- .../service/include/service/pythonservice.h | 16 - 20 files changed, 1277 insertions(+), 1019 deletions(-) create mode 100644 plugins/python/parser/include/pythonparser/pythonpersistence.h create mode 100644 plugins/python/parser/src/pythonpersistence.cpp delete mode 100644 plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py create mode 100644 plugins/python/parser/src/scripts/cc_python_parser/skip_file.py diff --git a/plugins/python/parser/CMakeLists.txt b/plugins/python/parser/CMakeLists.txt index 00a59fb63..b521ad072 100644 --- a/plugins/python/parser/CMakeLists.txt +++ b/plugins/python/parser/CMakeLists.txt @@ -10,7 +10,8 @@ include_directories( ${Python3_INCLUDE_DIRS}) add_library(pythonparser SHARED - src/pythonparser.cpp) + src/pythonparser.cpp + src/pythonpersistence.cpp) target_link_libraries(pythonparser util @@ -26,4 +27,6 @@ install(TARGETS pythonparser DESTINATION ${INSTALL_PARSER_DIR}) install( DIRECTORY ${PLUGIN_DIR}/parser/src/scripts/ DESTINATION ${INSTALL_PARSER_DIR}/scripts/python - FILES_MATCHING PATTERN "*.py") + FILES_MATCHING + PATTERN "*.py" + PATTERN "skip.json") diff --git a/plugins/python/parser/include/pythonparser/pythonpersistence.h b/plugins/python/parser/include/pythonparser/pythonpersistence.h new file mode 100644 index 000000000..5460c6eff --- /dev/null +++ b/plugins/python/parser/include/pythonparser/pythonpersistence.h @@ -0,0 +1,105 @@ +#ifndef CODECOMPASS_PYTHONPERSISTENCE_H +#define CODECOMPASS_PYTHONPERSISTENCE_H + +#include + +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cc { +namespace parser { + +class PythonPersistence { +private: + ParserContext &ctx; + +public: + PythonPersistence(ParserContext &ctx_) : ctx(ctx_) {} + + ~PythonPersistence(); + + void cppprint(boost::python::object o); + + void logInfo(boost::python::object o); + void logWarning(boost::python::object o); + void logError(boost::python::object o); + void logDebug(boost::python::object o); + + void persistFile(boost::python::object pyFile); + + void persistVariable(boost::python::object pyVariable); + + void persistFunction(boost::python::object pyFunction); + + void persistPreprocessedClass(boost::python::object pyClass); + + void persistClass(boost::python::object pyClass); + + void persistImport(boost::python::object pyImport); + +private: + boost::optional createFileLoc(boost::python::object filePosition); + + model::PythonEntityPtr getPythonEntity(const std::string &qualifiedName); + + model::PythonClassPtr getPythonClass(const std::string &qualifiedName); + + model::PythonVariablePtr getPythonVariable(const std::string &qualifiedName); + + bool isAstNodePersisted(const model::PythonAstNodePtr &node) const; + + bool isAstNodePersisted(const std::vector &nodes, + const model::PythonAstNodePtr &node) const; + + std::vector _astNodes; + std::vector _variables; + std::map > _variableUsages; + std::vector _functions; + std::map > _functionUsages; + std::vector _classes; + std::map > _classUsages; + std::vector _members; + std::vector _inheritance; + std::vector _imports; + std::vector _documentations; + std::vector _types; +}; + +typedef boost::shared_ptr PythonPersistencePtr; + +} +} + +#endif //CODECOMPASS_PYTHONPERSISTENCE_H diff --git a/plugins/python/parser/src/pythonparser.cpp b/plugins/python/parser/src/pythonparser.cpp index b23e5adda..a23b03ee8 100644 --- a/plugins/python/parser/src/pythonparser.cpp +++ b/plugins/python/parser/src/pythonparser.cpp @@ -4,895 +4,30 @@ #include #include #include -#include - -#include #include -#include -#include - -#include - -#include -#include -#include -#include +#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include namespace cc { namespace parser{ -void print(const std::string& s) { - std::cout << s << std::endl; -} - -class Persistence -{ -private: - ParserContext& ctx; - -public: - Persistence(ParserContext& ctx_) : ctx(ctx_) {} - ~Persistence(); - - void cppprint(boost::python::object o); - void persistFile(boost::python::object pyFile); - void persistVariable(boost::python::object pyVariable); - void persistFunction(boost::python::object pyFunction); - void persistPreprocessedClass(boost::python::object pyClass); - void persistClass(boost::python::object pyClass); - void persistImport(boost::python::object pyImport); - -private: - boost::optional createFileLocFromPythonFilePosition(boost::python::object filePosition); - model::PythonEntityPtr getPythonEntity(const std::string& qualifiedName); - model::PythonClassPtr getPythonClass(const std::string& qualifiedName); - model::PythonVariablePtr getPythonVariable(const std::string& qualifiedName); - bool isAstNodePersisted(const model::PythonAstNodePtr& node) const; - bool isAstNodePersisted(const std::vector& nodes, const model::PythonAstNodePtr& node) const; - - std::vector _astNodes; - std::vector _variables; - std::map> _variableUsages; - std::vector _functions; - std::map> _functionUsages; - std::vector _classes; - std::map> _classUsages; - std::vector _members; - std::vector _inheritance; - std::vector _imports; - std::vector _documentations; - std::vector _types; -}; - -template -static std::vector getKeys(const std::map& map) { - std::vector ret; - std::for_each(map.begin(), map.end(), [&ret](const auto& p){ ret.push_back(p.first); }); - return ret; -} - -Persistence::~Persistence() -{ - std::map m; - std::vector v = getKeys(m); - ctx.srcMgr.persistFiles(); - - (util::OdbTransaction(ctx.db))([this]{ - util::persistAll(_astNodes, ctx.db); - for(auto& ast : _variableUsages){ - util::persistAll(ast.second, ctx.db); - } - for(auto& ast : _functionUsages){ - util::persistAll(ast.second, ctx.db); - } - for(auto& ast : _classUsages){ - util::persistAll(ast.second, ctx.db); - } - util::persistAll(_variables, ctx.db); - util::persistAll(_functions, ctx.db); - util::persistAll(_classes, ctx.db); - util::persistAll(_members, ctx.db); - util::persistAll(_inheritance, ctx.db); - util::persistAll(_imports, ctx.db); - util::persistAll(_documentations, ctx.db); - util::persistAll(_types, ctx.db); - }); -} - -void Persistence::cppprint(boost::python::object o) { - std::string s = boost::python::extract(o); - std::cout << s << std::endl; -} - -void Persistence::persistFile(boost::python::object pyFile) -{ - try{ - model::FilePtr file = nullptr; - model::BuildSource buildSource; - - boost::python::object path = pyFile.attr("path"); - boost::python::object status = pyFile.attr("parse_status"); - - if(status.is_none()){ - std::cout << "status is None..." << std::endl; - } else if(path.is_none()){ - std::cout << "path is None..." << std::endl; - } else { - file = ctx.srcMgr.getFile(boost::python::extract(path)); - file->type = "PY"; - buildSource.file = file; - switch(boost::python::extract(status)){ - case 0: - buildSource.file->parseStatus = model::File::PSNone; - break; - case 1: - buildSource.file->parseStatus = model::File::PSPartiallyParsed; - break; - case 2: - buildSource.file->parseStatus = model::File::PSFullyParsed; - break; - default: - std::cout << "Unknown status: " << boost::python::extract(status) << std::endl; - } - - model::BuildActionPtr buildAction(new model::BuildAction); - buildAction->command = ""; - buildAction->type = model::BuildAction::Other; - try{ - util::OdbTransaction transaction{ctx.db}; - - transaction([&, this] { - ctx.db->persist(buildAction); - }); - - buildSource.action = buildAction; - - ctx.srcMgr.updateFile(*buildSource.file); - - transaction([&, this] { - ctx.db->persist(buildSource); - }); - } catch(const std::exception& ex){ - std::cout << "Exception: " << ex.what() << " - " << typeid(ex).name() << std::endl; - } - } - } catch(std::exception e){ - std::cout << e.what() << std::endl; - } -} - -void Persistence::persistVariable(boost::python::object pyVariable) -{ - try{ - boost::python::object name = pyVariable.attr("name"); - boost::python::object qualifiedName = pyVariable.attr("qualified_name"); - boost::python::object visibility = pyVariable.attr("visibility"); - - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyVariable.attr("file_position")); - - boost::python::list types = boost::python::extract(pyVariable.attr("type")); - - boost::python::list usages = boost::python::extract(pyVariable.attr("usages")); - - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || - types.is_none() || usages.is_none()){ - return; - } - - model::PythonAstNodePtr varAstNode(new model::PythonAstNode); - varAstNode->location = fileLoc.get(); - varAstNode->qualifiedName = boost::python::extract(qualifiedName); - varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; - varAstNode->astType = model::PythonAstNode::AstType::Declaration; - - varAstNode->id = model::createIdentifier(*varAstNode); - - if(isAstNodePersisted(varAstNode)){ - return; - } - - _astNodes.push_back(varAstNode); - model::PythonVariablePtr variable(new model::PythonVariable); - variable->astNodeId = varAstNode->id; - variable->name = boost::python::extract(name); - variable->qualifiedName = boost::python::extract(qualifiedName); - variable->visibility = boost::python::extract(visibility); - - variable->id = model::createIdentifier(*variable); - _variables.push_back(variable); - - for(int i = 0; i(types[i]); - model::PythonEntityPtr t = getPythonEntity(boost::python::extract(types[i])); - if(t == nullptr){ - continue; - } - type->type = t->id; - type->symbol = variable->id; - _types.push_back(type); - } - - _variableUsages[variable->id] = {}; - for(int i = 0; i usageFileLoc = - createFileLocFromPythonFilePosition(usages[i].attr("file_position")); - if(usageFileLoc == boost::none){ - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_variableUsages[variable->id], usageAstNode)) { - continue; - } - - _variableUsages[variable->id].push_back(usageAstNode); - } - - } catch (const odb::object_already_persistent& ex) - { - std::cout << "Var exception already persistent: " << ex.what() << std::endl; - } catch (const odb::database_exception& ex) - { - std::cout << "Var exception db exception: " << ex.what() << std::endl; - } catch(std::exception ex){ - std::cout << "Var exception: " << ex.what() << std::endl; - } -} - -void Persistence::persistFunction(boost::python::object pyFunction) -{ - try{ - boost::python::object name = pyFunction.attr("name"); - boost::python::object qualifiedName = pyFunction.attr("qualified_name"); - boost::python::object visibility = pyFunction.attr("visibility"); - - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyFunction.attr("file_position")); - - boost::python::list types = boost::python::extract(pyFunction.attr("type")); - - boost::python::list usages = boost::python::extract(pyFunction.attr("usages")); - - boost::python::object pyDocumentation = pyFunction.attr("documentation"); - - boost::python::list params = boost::python::extract(pyFunction.attr("parameters")); - - boost::python::list locals = boost::python::extract(pyFunction.attr("locals")); - - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || types.is_none() || - usages.is_none() || pyDocumentation.is_none() || params.is_none() || locals.is_none()){ - return; - } - - model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); - funcAstNode->location = fileLoc.get(); - funcAstNode->qualifiedName = boost::python::extract(qualifiedName); - funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; - funcAstNode->astType = model::PythonAstNode::AstType::Declaration; - - funcAstNode->id = model::createIdentifier(*funcAstNode); - - if(isAstNodePersisted(funcAstNode)){ - return; - } - - _astNodes.push_back(funcAstNode); - - model::PythonFunctionPtr function(new model::PythonFunction); - function->astNodeId = funcAstNode->id; - function->name = boost::python::extract(name); - function->qualifiedName = boost::python::extract(qualifiedName); - function->visibility = boost::python::extract(visibility); - - function->id = model::createIdentifier(*function); - - for(int i = 0; i(params[i])); - if(param == nullptr){ - continue; - } - function->parameters.push_back(param); - } - - for(int i = 0; i(locals[i])); - if(local == nullptr){ - continue; - } - function->locals.push_back(local); - } - - _functions.push_back(function); - - model::PythonDocumentationPtr documentation(new model::PythonDocumentation); - documentation->documentation = boost::python::extract(pyDocumentation); - documentation->documented = function->id; - documentation->documentationKind = model::PythonDocumentation::Function; - - _documentations.push_back(documentation); - - for(int i = 0; i(types[i])); - if(t == nullptr){ - continue; - } - type->type = t->id; - type->symbol = function->id; - - _types.push_back(type); - } - - _functionUsages[function->id] = {}; - for(int i = 0; i usageFileLoc = - createFileLocFromPythonFilePosition(usages[i].attr("file_position")); - if(usageFileLoc == boost::none){ - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_functionUsages[function->id], usageAstNode)) { - continue; - } - - _functionUsages[function->id].push_back(usageAstNode); - } - } catch(std::exception e){ - std::cout << "Func exception:" << e.what() << std::endl; - } -} - -void Persistence::persistPreprocessedClass(boost::python::object pyClass) +BOOST_PYTHON_MODULE(persistence) { - try{ - boost::python::object name = pyClass.attr("name"); - boost::python::object qualifiedName = pyClass.attr("qualified_name"); - boost::python::object visibility = pyClass.attr("visibility"); - - boost::optional fileLoc = createFileLocFromPythonFilePosition(pyClass.attr("file_position")); - - if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none){ - return; - } - - model::PythonAstNodePtr classAstNode(new model::PythonAstNode); - classAstNode->location = fileLoc.get(); - classAstNode->qualifiedName = boost::python::extract(qualifiedName); - classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; - classAstNode->astType = model::PythonAstNode::AstType::Declaration; - - classAstNode->id = model::createIdentifier(*classAstNode); - - if(isAstNodePersisted(classAstNode)){ - return; - } - - _astNodes.push_back(classAstNode); - - model::PythonClassPtr cl(new model::PythonClass); - cl->astNodeId = classAstNode->id; - cl->name = boost::python::extract(name); - cl->qualifiedName = boost::python::extract(qualifiedName); - cl->visibility = boost::python::extract(visibility); - - cl->id = model::createIdentifier(*cl); - - _classes.push_back(cl); - - } catch(std::exception e){ - std::cout << "Preprocessed class exception:" << e.what() << std::endl; - } -} - -void Persistence::persistClass(boost::python::object pyClass) -{ - try{ - boost::python::object qualifiedName = pyClass.attr("qualified_name"); - - boost::python::list usages = boost::python::extract(pyClass.attr("usages")); - - boost::python::object pyDocumentation = pyClass.attr("documentation"); - - boost::python::list baseClasses = boost::python::extract(pyClass.attr("base_classes")); - - boost::python::object members = pyClass.attr("members"); - - if(qualifiedName.is_none() || usages.is_none() || pyDocumentation.is_none() || - baseClasses.is_none() || members.is_none()){ - return; - } - - model::PythonClassPtr cl = getPythonClass(boost::python::extract(qualifiedName)); - - if (cl == nullptr){ - std::cout << "cl is none" << std::endl; - return; - } - - model::PythonDocumentationPtr documentation(new model::PythonDocumentation); - documentation->documentation = boost::python::extract(pyDocumentation); - documentation->documented = cl->id; - documentation->documentationKind = model::PythonDocumentation::Class; - - _documentations.push_back(documentation); - - _classUsages[cl->id] = {}; - for(int i = 0; i usageFileLoc = - createFileLocFromPythonFilePosition(usages[i].attr("file_position")); - if(usageFileLoc == boost::none){ - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_classUsages[cl->id], usageAstNode)) { - continue; - } - - _classUsages[cl->id].push_back(usageAstNode); - } - - for(int i = 0; iderived = cl->id; - std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); - - inheritance->base = getPythonEntity(baseClassQualifiedName)->id; - - _inheritance.push_back(inheritance); - } - - boost::python::list methods = boost::python::extract(members.attr("methods")); - boost::python::list staticMethods = boost::python::extract(members.attr("static_methods")); - boost::python::list attributes = boost::python::extract(members.attr("attributes")); - boost::python::list staticAttributes = boost::python::extract(members.attr("static_attributes")); - boost::python::list classes = boost::python::extract(members.attr("classes")); - - if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || - staticAttributes.is_none() || classes.is_none()){ - return; - } - - for(int i = 0; i(methods[i].attr("qualified_name")); - model::PythonEntityPtr method = getPythonEntity(qualifiedName); - if(method == nullptr){ - std::cout << "method is none" << std::endl; - continue; - } - classMember->astNodeId = method->astNodeId; - classMember->memberId = method->id; - classMember->classId = cl->id; - classMember->kind = model::PythonClassMember::Method; - classMember->staticMember = false; - - _members.push_back(classMember); - - boost::python::list _usages = boost::python::extract(methods[i].attr("usages")); - std::vector& _funcUsages = _functionUsages[method->id]; - for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); - if(_fl == boost::none || - std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _funcUsages.end()) - { - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = _fl.get(); - usageAstNode->qualifiedName = qualifiedName; - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_funcUsages, usageAstNode)) { - continue; - } - - _funcUsages.push_back(usageAstNode); - } - } - - for(int i = 0; i(staticMethods[i].attr("qualified_name")); - model::PythonEntityPtr method = getPythonEntity(qualifiedName); - if(method == nullptr){ - std::cout << "static method is none" << std::endl; - continue; - } - classMember->astNodeId = method->astNodeId; - classMember->memberId = method->id; - classMember->classId = cl->id; - classMember->kind = model::PythonClassMember::Method; - classMember->staticMember = true; - - _members.push_back(classMember); - - boost::python::list _usages = boost::python::extract(staticMethods[i].attr("usages")); - std::vector& _funcUsages = _functionUsages[method->id]; - for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); - if(_fl == boost::none || - std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _funcUsages.end()) - { - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = _fl.get(); - usageAstNode->qualifiedName = qualifiedName; - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_funcUsages, usageAstNode)) { - continue; - } - - _funcUsages.push_back(usageAstNode); - } - } - - for(int i = 0; i(attributes[i].attr("qualified_name")); - if (qualifiedName.empty()){ - continue; // TODO: import symbol in class - } - model::PythonEntityPtr attr = getPythonEntity(qualifiedName); - if(attr == nullptr){ - std::cout << "attr is none" << std::endl; - continue; - } - classMember->astNodeId = attr->astNodeId; - classMember->memberId = attr->id; - classMember->classId = cl->id; - classMember->kind = model::PythonClassMember::Attribute; - classMember->staticMember = false; - - _members.push_back(classMember); - - boost::python::list _usages = boost::python::extract(attributes[i].attr("usages")); - std::vector& _varUsages = _variableUsages[attr->id]; - for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); - if(_fl == boost::none || - std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _varUsages.end()) - { - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = _fl.get(); - usageAstNode->qualifiedName = qualifiedName; - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_varUsages, usageAstNode)) { - continue; - } - - _varUsages.push_back(usageAstNode); - } - } - - for(int i = 0; i(staticAttributes[i].attr("qualified_name")); - model::PythonEntityPtr attr = getPythonEntity(qualifiedName); - if(attr == nullptr){ - std::cout << "static attr is none" << std::endl; - continue; - } - classMember->astNodeId = attr->astNodeId; - classMember->memberId = attr->id; - classMember->classId = cl->id; - classMember->kind = model::PythonClassMember::Attribute; - classMember->staticMember = true; - - _members.push_back(classMember); - - boost::python::list _usages = boost::python::extract(staticAttributes[i].attr("usages")); - std::vector& _varUsages = _variableUsages[attr->id]; - for(int j = 0; j _fl = createFileLocFromPythonFilePosition(_usages[j].attr("file_position")); - if(_fl == boost::none || - std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p){ - return p->location.file == _fl.get().file && p->location.range == _fl.get().range; }) != _varUsages.end()) - { - continue; - } - model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); - usageAstNode->location = _fl.get(); - usageAstNode->qualifiedName = qualifiedName; - usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; - usageAstNode->astType = model::PythonAstNode::AstType::Usage; - - usageAstNode->id = model::createIdentifier(*usageAstNode); - - if(isAstNodePersisted(_varUsages, usageAstNode)) { - continue; - } - - _varUsages.push_back(usageAstNode); - } - } - - for(int i = 0; i(classes[i].attr("qualified_name")); - model::PythonEntityPtr inner = getPythonEntity(qualifiedName); - if(inner == nullptr){ - std::cout << "inner class is none" << std::endl; - continue; - } - classMember->astNodeId = inner->astNodeId; - classMember->memberId = inner->id; - classMember->classId = cl->id; - classMember->kind = model::PythonClassMember::Class; - classMember->staticMember = false; - - _members.push_back(classMember); - } - } catch(std::exception e){ - std::cout << "Class exception:" << e.what() << std::endl; - } -} - -void Persistence::persistImport(boost::python::object pyImport) -{ - try { - model::FilePtr file = ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); - - boost::python::list importedModules = - boost::python::extract(pyImport.attr("imported_modules")); - - boost::python::dict importedSymbols = - boost::python::extract(pyImport.attr("imported_symbols")); - - if (file == nullptr || importedModules.is_none() || importedSymbols.is_none()) { - return; - } - - for (int i = 0; i < boost::python::len(importedModules); ++i) { - boost::python::object importData = importedModules[i]; - - model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); - boost::optional fileLoc = createFileLocFromPythonFilePosition(importData.attr("position")); - std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); - - if (moduleFile == nullptr || fileLoc == boost::none) { - continue; - } - - model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); - moduleAstNode->location = fileLoc.get(); - moduleAstNode->qualifiedName = qualifiedName; - moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; - moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; - - moduleAstNode->id = model::createIdentifier(*moduleAstNode); - - if(isAstNodePersisted(moduleAstNode)){ - continue; - } - - model::PythonImportPtr moduleImport(new model::PythonImport); - moduleImport->astNodeId = moduleAstNode->id; - moduleImport->importer = file; - moduleImport->imported = moduleFile; - - _astNodes.push_back(moduleAstNode); - _imports.push_back(moduleImport); - } - - boost::python::list importDict = importedSymbols.items(); - for (int i = 0; i < boost::python::len(importDict); ++i) { - boost::python::tuple import = boost::python::extract(importDict[i]); - - boost::python::object importData = import[0]; - - model::FilePtr moduleFile = ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); - boost::optional fileLoc = createFileLocFromPythonFilePosition(importData.attr("position")); - std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); - - if (moduleFile == nullptr || fileLoc == boost::none) { - continue; - } - - model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); - moduleAstNode->location = fileLoc.get(); - moduleAstNode->qualifiedName = qualifiedName; - moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; - moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; - - moduleAstNode->id = model::createIdentifier(*moduleAstNode); - - if(isAstNodePersisted(moduleAstNode)){ - continue; - } - - for (int j = 0; j < boost::python::len(import[1]); ++j){ - model::PythonImportPtr moduleImport(new model::PythonImport); - moduleImport->astNodeId = moduleAstNode->id; - moduleImport->importer = file; - moduleImport->imported = moduleFile; - auto symb = getPythonEntity(boost::python::extract(import[1][j])); - if(symb == nullptr){ - continue; - } - moduleImport->importedSymbol = symb->id; - - _imports.push_back(moduleImport); - } - - _astNodes.push_back(moduleAstNode); - } - } catch(std::exception e){ - std::cout << "Import exception:" << e.what() << std::endl; - } -} - -boost::optional Persistence::createFileLocFromPythonFilePosition(boost::python::object filePosition) -{ - if (filePosition.is_none()){ - return boost::none; - } - - boost::python::object filePath = filePosition.attr("file"); - boost::python::object pyRange = filePosition.attr("range"); - - if (filePath.is_none() || pyRange.is_none()){ - return boost::none; - } - - boost::python::object pyStartPosition = pyRange.attr("start_position"); - boost::python::object pyEndPosition = pyRange.attr("end_position"); - - if (pyStartPosition.is_none() || pyEndPosition.is_none()){ - return boost::none; - } - - model::FileLoc fileLoc; - - fileLoc.file = ctx.srcMgr.getFile(boost::python::extract(filePath)); - - model::Position startPosition(boost::python::extract(pyStartPosition.attr("line")), - boost::python::extract(pyStartPosition.attr("column"))); - model::Position endPosition(boost::python::extract(pyEndPosition.attr("line")), - boost::python::extract(pyEndPosition.attr("column"))); - - fileLoc.range = model::Range(startPosition, endPosition); - - return fileLoc; -} - -model::PythonEntityPtr Persistence::getPythonEntity(const std::string& qualifiedName) -{ - if(qualifiedName.empty()){ - return nullptr; - } - - auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); - if (varIt != _variables.end()){ - return *varIt; - } - - auto funcIt = std::find_if(_functions.begin(), _functions.end(), [&](const auto& func){ return func->qualifiedName == qualifiedName; }); - if (funcIt != _functions.end()){ - return *funcIt; - } - - auto classIt = std::find_if(_classes.begin(), _classes.end(), [&](const auto& cl){ return cl->qualifiedName == qualifiedName; }); - if (classIt != _classes.end()){ - return *classIt; - } - - return nullptr; -} - -model::PythonVariablePtr Persistence::getPythonVariable(const std::string& qualifiedName) -{ - if(qualifiedName.empty()){ - return nullptr; - } - - auto varIt = std::find_if(_variables.begin(), _variables.end(), [&](const auto& var){ return var->qualifiedName == qualifiedName; }); - if (varIt != _variables.end()){ - return *varIt; - } - - return nullptr; -} - -model::PythonClassPtr Persistence::getPythonClass(const std::string& qualifiedName) -{ - auto classIt = std::find_if(_classes.begin(), _classes.end(), [&](const auto& cl){ return cl->qualifiedName == qualifiedName; }); - if (classIt != _classes.end()){ - return *classIt; - } - - return nullptr; -} - -bool Persistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const -{ - for(auto it : _astNodes){ - if(*it == *node){ - return true; - } - } - - return false; -} - -bool Persistence::isAstNodePersisted(const std::vector& nodes, const model::PythonAstNodePtr& node) const -{ - for(auto it : nodes){ - if(*it == *node){ - return true; - } - } - - return false; -} - -typedef boost::shared_ptr PersistencePtr; - -BOOST_PYTHON_MODULE(persistence){ - boost::python::class_("Persistence", boost::python::init()) - .def("print", &Persistence::cppprint) - .def("persist_file", &Persistence::persistFile) - .def("persist_variable", &Persistence::persistVariable) - .def("persist_function", &Persistence::persistFunction) - .def("persist_preprocessed_class", &Persistence::persistPreprocessedClass) - .def("persist_class", &Persistence::persistClass) - .def("persist_import", &Persistence::persistImport); + boost::python::class_("Persistence", boost::python::init()) + .def("print", &PythonPersistence::cppprint) + .def("log_info", &PythonPersistence::logInfo) + .def("log_warning", &PythonPersistence::logWarning) + .def("log_error", &PythonPersistence::logError) + .def("log_debug", &PythonPersistence::logDebug) + .def("persist_file", &PythonPersistence::persistFile) + .def("persist_variable", &PythonPersistence::persistVariable) + .def("persist_function", &PythonPersistence::persistFunction) + .def("persist_preprocessed_class", &PythonPersistence::persistPreprocessedClass) + .def("persist_class", &PythonPersistence::persistClass) + .def("persist_import", &PythonPersistence::persistImport); } PythonParser::PythonParser(ParserContext &ctx_) : AbstractParser(ctx_) {} @@ -908,7 +43,8 @@ bool PythonParser::cleanupDatabase() { return true; } bool PythonParser::parse() { const std::string PARSER_SCRIPTS_DIR = _ctx.compassRoot + "/lib/parserplugin/scripts/python"; - if (!boost::filesystem::exists(PARSER_SCRIPTS_DIR) || !boost::filesystem::is_directory(PARSER_SCRIPTS_DIR)){ + if (!boost::filesystem::exists(PARSER_SCRIPTS_DIR) || !boost::filesystem::is_directory(PARSER_SCRIPTS_DIR)) + { throw std::runtime_error(PARSER_SCRIPTS_DIR + " is not a directory!"); } @@ -920,34 +56,39 @@ bool PythonParser::parse() boost::python::object module = boost::python::import("cc_python_parser.python_parser"); - if(!module.is_none()){ + if(!module.is_none()) + { boost::python::object func = module.attr("parse"); - if(!func.is_none() && PyCallable_Check(func.ptr())){ + if(!func.is_none() && PyCallable_Check(func.ptr())) + { std::string source_path; - for (const std::string& input : _ctx.options["input"].as>()){ - if (boost::filesystem::is_directory(input)){ + for (const std::string& input : _ctx.options["input"].as>()) + { + if (boost::filesystem::is_directory(input)) + { source_path = input; } } - if(source_path.empty()){ - std::cout << "No source path was found" << std::endl; + if(source_path.empty()) + { + LOG(error) << "No source path was found"; } else { - PersistencePtr persistencePtr(new Persistence(_ctx)); + PythonPersistencePtr persistencePtr(new PythonPersistence(_ctx)); func(source_path, boost::python::ptr(persistencePtr.get())); } } else { - std::cout << "Cannot find function" << std::endl; + LOG(error) << "Cannot find function"; } } else { - std::cout << "Cannot import module" << std::endl; + LOG(error) << "Cannot import module"; } - }catch(boost::python::error_already_set){ + } catch (boost::python::error_already_set) + { PyErr_Print(); } - // Py_Finalize(); return true; } diff --git a/plugins/python/parser/src/pythonpersistence.cpp b/plugins/python/parser/src/pythonpersistence.cpp new file mode 100644 index 000000000..e81968127 --- /dev/null +++ b/plugins/python/parser/src/pythonpersistence.cpp @@ -0,0 +1,993 @@ +#include + +#include + + +namespace cc +{ +namespace parser +{ + + +void print(const std::string& s) +{ + LOG(info) << s; +} + +template +static std::vector getKeys(const std::map& map) +{ + std::vector ret; + std::for_each(map.begin(), map.end(), [&ret](const auto& p) + { ret.push_back(p.first); }); + return ret; +} + +PythonPersistence::~PythonPersistence() +{ + std::map m; + std::vector v = getKeys(m); + ctx.srcMgr.persistFiles(); + + (util::OdbTransaction(ctx.db))([this]{ + util::persistAll(_astNodes, ctx.db); + for(auto& ast : _variableUsages) + { + util::persistAll(ast.second, ctx.db); + } + for(auto& ast : _functionUsages) + { + util::persistAll(ast.second, ctx.db); + } + for(auto& ast : _classUsages) + { + util::persistAll(ast.second, ctx.db); + } + util::persistAll(_variables, ctx.db); + util::persistAll(_functions, ctx.db); + util::persistAll(_classes, ctx.db); + util::persistAll(_members, ctx.db); + util::persistAll(_inheritance, ctx.db); + util::persistAll(_imports, ctx.db); + util::persistAll(_documentations, ctx.db); + util::persistAll(_types, ctx.db); + }); +} + +void PythonPersistence::cppprint(boost::python::object o) +{ + std::string s = boost::python::extract(o); + LOG(info) << s; +} + +void PythonPersistence::logInfo(boost::python::object o) +{ + std::string s = boost::python::extract(o); + LOG(info) << s; +} + +void PythonPersistence::logWarning(boost::python::object o) +{ + std::string s = boost::python::extract(o); + LOG(warning) << s; +} + +void PythonPersistence::logError(boost::python::object o) +{ + std::string s = boost::python::extract(o); + LOG(error) << s; +} + +void PythonPersistence::logDebug(boost::python::object o) +{ + std::string s = boost::python::extract(o); + LOG(debug) << s; +} + +void PythonPersistence::persistFile(boost::python::object pyFile) +{ + try{ + model::FilePtr file = nullptr; + model::BuildSource buildSource; + + boost::python::object path = pyFile.attr("path"); + boost::python::object status = pyFile.attr("parse_status"); + + if(status.is_none()) + { + LOG(error) << "Persist file: status is None..."; + } else if(path.is_none()) + { + LOG(error) << "Persist file: path is None..."; + } else { + file = ctx.srcMgr.getFile(boost::python::extract(path)); + file->type = "PY"; + buildSource.file = file; + switch(boost::python::extract(status)) + { + case 0: + buildSource.file->parseStatus = model::File::PSNone; + break; + case 1: + buildSource.file->parseStatus = model::File::PSPartiallyParsed; + break; + case 2: + buildSource.file->parseStatus = model::File::PSFullyParsed; + break; + default: + LOG(error) << "Persist file unknown status: " << boost::python::extract(status); + } + + model::BuildActionPtr buildAction(new model::BuildAction); + buildAction->command = ""; + buildAction->type = model::BuildAction::Other; + try{ + util::OdbTransaction transaction{ctx.db}; + + transaction([&, this] { + ctx.db->persist(buildAction); + }); + + buildSource.action = buildAction; + + ctx.srcMgr.updateFile(*buildSource.file); + + transaction([&, this] { + ctx.db->persist(buildSource); + }); + } catch(const std::exception& ex) + { + LOG(error) << "Persist file exception: " << ex.what() << " - " << typeid(ex).name(); + } + } + } catch(std::exception e) + { + LOG(error) << "Persist file exception: " << e.what(); + } +} + +void PythonPersistence::persistVariable(boost::python::object pyVariable) +{ + try{ + boost::python::object name = pyVariable.attr("name"); + boost::python::object qualifiedName = pyVariable.attr("qualified_name"); + boost::python::object visibility = pyVariable.attr("visibility"); + + boost::optional fileLoc = + createFileLoc(pyVariable.attr("file_position")); + + boost::python::list types = + boost::python::extract(pyVariable.attr("type")); + + boost::python::list usages = + boost::python::extract(pyVariable.attr("usages")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || + types.is_none() || usages.is_none()) + { + return; + } + + model::PythonAstNodePtr varAstNode(new model::PythonAstNode); + varAstNode->location = fileLoc.get(); + varAstNode->qualifiedName = boost::python::extract(qualifiedName); + varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + varAstNode->astType = model::PythonAstNode::AstType::Declaration; + + varAstNode->id = model::createIdentifier(*varAstNode); + + if(isAstNodePersisted(varAstNode)) + { + return; + } + + _astNodes.push_back(varAstNode); + model::PythonVariablePtr variable(new model::PythonVariable); + variable->astNodeId = varAstNode->id; + variable->name = boost::python::extract(name); + variable->qualifiedName = boost::python::extract(qualifiedName); + variable->visibility = boost::python::extract(visibility); + + variable->id = model::createIdentifier(*variable); + _variables.push_back(variable); + + for(int i = 0; i(types[i]); + model::PythonEntityPtr t = + getPythonEntity(boost::python::extract(types[i])); + if(!t) + { + continue; + } + type->type = t->id; + type->symbol = variable->id; + _types.push_back(type); + } + + _variableUsages[variable->id] = {}; + for(int i = 0; i usageFileLoc = + createFileLoc(usages[i].attr("file_position")); + if(usageFileLoc == boost::none) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_variableUsages[variable->id], usageAstNode)) + { + continue; + } + + _variableUsages[variable->id].push_back(usageAstNode); + } + + } catch (const odb::object_already_persistent& ex) + { + LOG(error) << "Persist variable exception already persistent: " << ex.what() << std::endl; + } catch (const odb::database_exception& ex) + { + LOG(error) << "Persist variable exception db exception: " << ex.what(); + } catch(std::exception ex) + { + LOG(error) << "Persist variable exception: " << ex.what(); + } +} + +void PythonPersistence::persistFunction(boost::python::object pyFunction) +{ + try{ + boost::python::object name = pyFunction.attr("name"); + boost::python::object qualifiedName = pyFunction.attr("qualified_name"); + boost::python::object visibility = pyFunction.attr("visibility"); + + boost::optional fileLoc = + createFileLoc(pyFunction.attr("file_position")); + + boost::python::list types = + boost::python::extract(pyFunction.attr("type")); + + boost::python::list usages = + boost::python::extract(pyFunction.attr("usages")); + + boost::python::object pyDocumentation = pyFunction.attr("documentation"); + + boost::python::list params = + boost::python::extract(pyFunction.attr("parameters")); + + boost::python::list locals = + boost::python::extract(pyFunction.attr("locals")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || types.is_none() || + usages.is_none() || pyDocumentation.is_none() || params.is_none() || locals.is_none()) + { + return; + } + + model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); + funcAstNode->location = fileLoc.get(); + funcAstNode->qualifiedName = boost::python::extract(qualifiedName); + funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + funcAstNode->astType = model::PythonAstNode::AstType::Declaration; + + funcAstNode->id = model::createIdentifier(*funcAstNode); + + if(isAstNodePersisted(funcAstNode)) + { + return; + } + + _astNodes.push_back(funcAstNode); + + model::PythonFunctionPtr function(new model::PythonFunction); + function->astNodeId = funcAstNode->id; + function->name = boost::python::extract(name); + function->qualifiedName = boost::python::extract(qualifiedName); + function->visibility = boost::python::extract(visibility); + + function->id = model::createIdentifier(*function); + + for(int i = 0; i(params[i])); + if(!param) + { + continue; + } + function->parameters.push_back(param); + } + + for(int i = 0; i(locals[i])); + if(!local) + { + continue; + } + function->locals.push_back(local); + } + + _functions.push_back(function); + + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = boost::python::extract(pyDocumentation); + documentation->documented = function->id; + documentation->documentationKind = model::PythonDocumentation::Function; + + _documentations.push_back(documentation); + + for(int i = 0; i(types[i])); + if(!t) + { + continue; + } + type->type = t->id; + type->symbol = function->id; + + _types.push_back(type); + } + + _functionUsages[function->id] = {}; + for(int i = 0; i usageFileLoc = + createFileLoc(usages[i].attr("file_position")); + if(usageFileLoc == boost::none) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_functionUsages[function->id], usageAstNode)) + { + continue; + } + + _functionUsages[function->id].push_back(usageAstNode); + } + } catch(std::exception e) + { + LOG(error) << "Persist function exception:" << e.what(); + } +} + +void PythonPersistence::persistPreprocessedClass(boost::python::object pyClass) +{ + try{ + boost::python::object name = pyClass.attr("name"); + boost::python::object qualifiedName = pyClass.attr("qualified_name"); + boost::python::object visibility = pyClass.attr("visibility"); + + boost::optional fileLoc = + createFileLoc(pyClass.attr("file_position")); + + if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none) + { + return; + } + + model::PythonAstNodePtr classAstNode(new model::PythonAstNode); + classAstNode->location = fileLoc.get(); + classAstNode->qualifiedName = boost::python::extract(qualifiedName); + classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + classAstNode->astType = model::PythonAstNode::AstType::Declaration; + + classAstNode->id = model::createIdentifier(*classAstNode); + + if(isAstNodePersisted(classAstNode)) + { + return; + } + + _astNodes.push_back(classAstNode); + + model::PythonClassPtr cl(new model::PythonClass); + cl->astNodeId = classAstNode->id; + cl->name = boost::python::extract(name); + cl->qualifiedName = boost::python::extract(qualifiedName); + cl->visibility = boost::python::extract(visibility); + + cl->id = model::createIdentifier(*cl); + + _classes.push_back(cl); + + } catch(std::exception e) + { + LOG(error) << "Persist preprocessed class exception:" << e.what(); + } +} + +void PythonPersistence::persistClass(boost::python::object pyClass) +{ + try{ + boost::python::object qualifiedName = pyClass.attr("qualified_name"); + + boost::python::list usages = + boost::python::extract(pyClass.attr("usages")); + + boost::python::object pyDocumentation = pyClass.attr("documentation"); + + boost::python::list baseClasses = + boost::python::extract(pyClass.attr("base_classes")); + + boost::python::object members = pyClass.attr("members"); + + if(qualifiedName.is_none() || usages.is_none() || pyDocumentation.is_none() || + baseClasses.is_none() || members.is_none()) + { + return; + } + + model::PythonClassPtr cl = + getPythonClass(boost::python::extract(qualifiedName)); + + if (!cl) + { + LOG(error) << "Persist class: cl is none"; + return; + } + + model::PythonDocumentationPtr documentation(new model::PythonDocumentation); + documentation->documentation = boost::python::extract(pyDocumentation); + documentation->documented = cl->id; + documentation->documentationKind = model::PythonDocumentation::Class; + + _documentations.push_back(documentation); + + _classUsages[cl->id] = {}; + for(int i = 0; i usageFileLoc = + createFileLoc(usages[i].attr("file_position")); + if(usageFileLoc == boost::none) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = usageFileLoc.get(); + usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_classUsages[cl->id], usageAstNode)) + { + continue; + } + + _classUsages[cl->id].push_back(usageAstNode); + } + + for(int i = 0; iderived = cl->id; + std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); + + inheritance->base = getPythonEntity(baseClassQualifiedName)->id; + + _inheritance.push_back(inheritance); + } + + boost::python::list methods = + boost::python::extract(members.attr("methods")); + boost::python::list staticMethods = + boost::python::extract(members.attr("static_methods")); + boost::python::list attributes = + boost::python::extract(members.attr("attributes")); + boost::python::list staticAttributes = + boost::python::extract(members.attr("static_attributes")); + boost::python::list classes = + boost::python::extract(members.attr("classes")); + + if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || + staticAttributes.is_none() || classes.is_none()) + { + return; + } + + for(int i = 0; i(methods[i].attr("qualified_name")); + model::PythonEntityPtr method = getPythonEntity(qualifiedName); + if(!method) + { + LOG(error) << "Persist class: method is none"; + continue; + } + classMember->astNodeId = method->astNodeId; + classMember->memberId = method->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = false; + + _members.push_back(classMember); + + boost::python::list _usages = + boost::python::extract(methods[i].attr("usages")); + std::vector& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_funcUsages.begin(), _funcUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + }) != _funcUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_funcUsages, usageAstNode)) + { + continue; + } + + _funcUsages.push_back(usageAstNode); + } + } + + for(int i = 0; i(staticMethods[i].attr("qualified_name")); + model::PythonEntityPtr method = getPythonEntity(qualifiedName); + if(!method) + { + LOG(error) << "Persist class: static method is none"; + continue; + } + classMember->astNodeId = method->astNodeId; + classMember->memberId = method->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Method; + classMember->staticMember = true; + + _members.push_back(classMember); + + boost::python::list _usages = + boost::python::extract(staticMethods[i].attr("usages")); + std::vector& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if (_funcUsages.begin(), _funcUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + } + ) != _funcUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_funcUsages, usageAstNode)) + { + continue; + } + + _funcUsages.push_back(usageAstNode); + } + } + + for(int i = 0; i(attributes[i].attr("qualified_name")); + if (qualifiedName.empty()) + { + continue; // TODO: import symbol in class + } + model::PythonEntityPtr attr = getPythonEntity(qualifiedName); + if(!attr) + { + LOG(error) << "Persist class: attr is none"; + continue; + } + classMember->astNodeId = attr->astNodeId; + classMember->memberId = attr->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = false; + + _members.push_back(classMember); + + boost::python::list _usages = + boost::python::extract(attributes[i].attr("usages")); + std::vector& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + }) != _varUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_varUsages, usageAstNode)) + { + continue; + } + + _varUsages.push_back(usageAstNode); + } + } + + for(int i = 0; i(staticAttributes[i].attr("qualified_name")); + model::PythonEntityPtr attr = getPythonEntity(qualifiedName); + if(!attr) + { + LOG(error) << "Persist class: static attr is none"; + continue; + } + classMember->astNodeId = attr->astNodeId; + classMember->memberId = attr->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Attribute; + classMember->staticMember = true; + + _members.push_back(classMember); + + boost::python::list _usages = + boost::python::extract(staticAttributes[i].attr("usages")); + std::vector& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); + if(_fl == boost::none || + std::find_if(_varUsages.begin(), _varUsages.end(), [&](const auto& p) + { + return p->location.file == _fl.get().file && p->location.range == _fl.get().range; + }) != _varUsages.end()) + { + continue; + } + model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); + usageAstNode->location = _fl.get(); + usageAstNode->qualifiedName = qualifiedName; + usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; + usageAstNode->astType = model::PythonAstNode::AstType::Usage; + + usageAstNode->id = model::createIdentifier(*usageAstNode); + + if(isAstNodePersisted(_varUsages, usageAstNode)) + { + continue; + } + + _varUsages.push_back(usageAstNode); + } + } + + for(int i = 0; i(classes[i].attr("qualified_name")); + model::PythonEntityPtr inner = getPythonEntity(qualifiedName); + if(!inner) + { + LOG(error) << "Persist class: inner class is none"; + continue; + } + classMember->astNodeId = inner->astNodeId; + classMember->memberId = inner->id; + classMember->classId = cl->id; + classMember->kind = model::PythonClassMember::Class; + classMember->staticMember = false; + + _members.push_back(classMember); + } + } catch(std::exception e) + { + LOG(error) << "Persist class exception:" << e.what(); + } +} + +void PythonPersistence::persistImport(boost::python::object pyImport) +{ + try { + model::FilePtr file = + ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); + + boost::python::list importedModules = + boost::python::extract(pyImport.attr("imported_modules")); + + boost::python::dict importedSymbols = + boost::python::extract(pyImport.attr("imported_symbols")); + + if (!file || importedModules.is_none() || importedSymbols.is_none()) + { + return; + } + + for (int i = 0; i < boost::python::len(importedModules); ++i) + { + boost::python::object importData = importedModules[i]; + + model::FilePtr moduleFile = + ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); + boost::optional fileLoc = createFileLoc(importData.attr("position")); + std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); + + if (!moduleFile || fileLoc == boost::none) + { + continue; + } + + model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); + moduleAstNode->location = fileLoc.get(); + moduleAstNode->qualifiedName = qualifiedName; + moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; + moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; + + moduleAstNode->id = model::createIdentifier(*moduleAstNode); + + if(isAstNodePersisted(moduleAstNode)) + { + continue; + } + + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->astNodeId = moduleAstNode->id; + moduleImport->importer = file; + moduleImport->imported = moduleFile; + + _astNodes.push_back(moduleAstNode); + _imports.push_back(moduleImport); + } + + boost::python::list importDict = importedSymbols.items(); + for (int i = 0; i < boost::python::len(importDict); ++i) + { + boost::python::tuple import = boost::python::extract(importDict[i]); + + boost::python::object importData = import[0]; + + model::FilePtr moduleFile = + ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); + boost::optional fileLoc = createFileLoc(importData.attr("position")); + std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); + + if (!moduleFile || fileLoc == boost::none) + { + continue; + } + + model::PythonAstNodePtr moduleAstNode(new model::PythonAstNode); + moduleAstNode->location = fileLoc.get(); + moduleAstNode->qualifiedName = qualifiedName; + moduleAstNode->symbolType = model::PythonAstNode::SymbolType::Module; + moduleAstNode->astType = model::PythonAstNode::AstType::Declaration; + + moduleAstNode->id = model::createIdentifier(*moduleAstNode); + + if(isAstNodePersisted(moduleAstNode)) + { + continue; + } + + for (int j = 0; j < boost::python::len(import[1]); ++j) + { + model::PythonImportPtr moduleImport(new model::PythonImport); + moduleImport->astNodeId = moduleAstNode->id; + moduleImport->importer = file; + moduleImport->imported = moduleFile; + auto symb = + getPythonEntity(boost::python::extract(import[1][j])); + if(!symb) + { + continue; + } + moduleImport->importedSymbol = symb->id; + + _imports.push_back(moduleImport); + } + + _astNodes.push_back(moduleAstNode); + } + } catch(std::exception e) + { + LOG(error) << "Persist import exception:" << e.what(); + } +} + +boost::optional PythonPersistence::createFileLoc(boost::python::object filePosition) +{ + if (filePosition.is_none()) + { + return boost::none; + } + + boost::python::object filePath = filePosition.attr("file"); + boost::python::object pyRange = filePosition.attr("range"); + + if (filePath.is_none() || pyRange.is_none()) + { + return boost::none; + } + + boost::python::object pyStartPosition = pyRange.attr("start_position"); + boost::python::object pyEndPosition = pyRange.attr("end_position"); + + if (pyStartPosition.is_none() || pyEndPosition.is_none()) + { + return boost::none; + } + + model::FileLoc fileLoc; + + fileLoc.file = ctx.srcMgr.getFile(boost::python::extract(filePath)); + + model::Position startPosition(boost::python::extract(pyStartPosition.attr("line")), + boost::python::extract(pyStartPosition.attr("column"))); + model::Position endPosition(boost::python::extract(pyEndPosition.attr("line")), + boost::python::extract(pyEndPosition.attr("column"))); + + fileLoc.range = model::Range(startPosition, endPosition); + + return fileLoc; +} + +model::PythonEntityPtr PythonPersistence::getPythonEntity(const std::string& qualifiedName) +{ + if(qualifiedName.empty()) + { + return nullptr; + } + + auto varIt = + std::find_if(_variables.begin(), + _variables.end(), + [&](const auto& var) + { return var->qualifiedName == qualifiedName; } + ); + if (varIt != _variables.end()) + { + return *varIt; + } + + auto funcIt = + std::find_if(_functions.begin(), + _functions.end(), + [&](const auto& func) + { return func->qualifiedName == qualifiedName; } + ); + if (funcIt != _functions.end()) + { + return *funcIt; + } + + auto classIt = + std::find_if(_classes.begin(), + _classes.end(), + [&](const auto& cl) + { return cl->qualifiedName == qualifiedName; } + ); + if (classIt != _classes.end()) + { + return *classIt; + } + + return nullptr; +} + +model::PythonVariablePtr PythonPersistence::getPythonVariable(const std::string& qualifiedName) +{ + if(qualifiedName.empty()) + { + return nullptr; + } + + auto varIt = + std::find_if(_variables.begin(), + _variables.end(), + [&](const auto& var) + { return var->qualifiedName == qualifiedName; } + ); + if (varIt != _variables.end()) + { + return *varIt; + } + + return nullptr; +} + +model::PythonClassPtr PythonPersistence::getPythonClass(const std::string& qualifiedName) +{ + auto classIt = + std::find_if(_classes.begin(), + _classes.end(), + [&](const auto& cl) + { return cl->qualifiedName == qualifiedName; } + ); + if (classIt != _classes.end()) + { + return *classIt; + } + + return nullptr; +} + +bool PythonPersistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const +{ + for(auto it : _astNodes) + { + if(*it == *node) + { + return true; + } + } + + return false; +} + +bool PythonPersistence::isAstNodePersisted( + const std::vector& nodes, + const model::PythonAstNodePtr& node) const +{ + for(auto it : nodes) + { + if(*it == *node) + { + return true; + } + } + + return false; +} + +} +} \ No newline at end of file diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py b/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py index 1df5a03e2..1f44934ab 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/__init__.py @@ -1 +1 @@ -__all__ = ["hashable_list", "file_position", "parser_tree", "position", "unique_list", "utils"] +__all__ = ["hashable_list", "file_position", "parser_tree", "position", "utils"] diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py b/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py index 91f6bdd11..48d0a8cc0 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/hashable_list.py @@ -1,20 +1,8 @@ -from collections import Counter from typing import List, TypeVar, Generic T = TypeVar('T') -class HashableList(Generic[T], List[T]): - def __hash__(self): - return hash(e for e in self) - - def __eq__(self, other): - return isinstance(other, type(self)) and Counter(self) == Counter(other) - - def __ne__(self, other): - return not self.__eq__(other) - - class OrderedHashableList(Generic[T], List[T]): def __hash__(self): return hash(e for e in self) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py b/plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py deleted file mode 100644 index 952f707ae..000000000 --- a/plugins/python/parser/src/scripts/cc_python_parser/common/unique_list.py +++ /dev/null @@ -1,23 +0,0 @@ -from typing import TypeVar, List, Iterable - -T = TypeVar('T') - - -class UniqueList(List[T]): - def __init__(self, seq=()): - super().__init__() - self.extend(seq) - - def append(self, obj: T) -> None: - if obj in self: - self.remove(obj) - super().append(obj) - - def extend(self, iterable: Iterable[T]) -> None: - for i in iterable: - self.append(i) - - def insert(self, index: int, obj: T) -> None: - if obj in self: - self.remove(obj) - super().insert(index, obj) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py index 8f680a204..9a8a4409d 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/common/utils.py @@ -3,12 +3,13 @@ from typing import Final, List, Callable from cc_python_parser.common.position import Range, Position +from cc_python_parser.persistence.persistence import ModelPersistence ENCODINGS: Final[List[str]] = ['utf-8', 'iso-8859-1'] -def process_file_content(path: PurePath, func: Callable[[str, int], None]) -> None: +def process_file_content(path: PurePath, func: Callable[[str, int], None], persistence: ModelPersistence) -> None: if not Path(path).exists(): return for encoding in ENCODINGS: @@ -20,11 +21,11 @@ def process_file_content(path: PurePath, func: Callable[[str, int], None]) -> No except UnicodeDecodeError: continue except FileNotFoundError: - print(f"File not found: {path}") + persistence.log_error(f"File not found: {path}") continue else: return - print(f"Unhandled encoding in {str(path)}") + persistence.log_error(f"Unhandled encoding in {str(path)}") def has_attr(obj, attrs) -> bool: diff --git a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py index a1f1d1685..c95f0f693 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/file_info.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/file_info.py @@ -7,6 +7,7 @@ from cc_python_parser.persistence.file_dto import FileDTO from cc_python_parser.preprocessed_file import PreprocessedFile from cc_python_parser.symbol_collector import SymbolCollector +from cc_python_parser.persistence.persistence import ModelPersistence @unique @@ -18,11 +19,12 @@ class ProcessStatus(Enum): # TODO: expr_1; import; expr_2; - correct script -> not a problem class FileInfo: - def __init__(self, path: PurePath): + def __init__(self, path: PurePath, persistence: ModelPersistence): self.path: PurePath = path self.symbol_collector: Optional[SymbolCollector] = None self.preprocessed_file: PreprocessedFile = PreprocessedFile(path) self.status: ProcessStatus = ProcessStatus.WAITING + self.persistence: ModelPersistence = persistence def get_file(self): return self.path.name @@ -49,15 +51,14 @@ def create_dto(self) -> FileDTO: file_dto.documentation = self.preprocessed_file.documentation return file_dto - @staticmethod - def get_content(file: PurePath) -> str: + def get_content(self, file: PurePath) -> str: content = "" def handle_file_content(c, _): nonlocal content content = c - process_file_content(file, handle_file_content) + process_file_content(file, handle_file_content, self.persistence) return content @staticmethod diff --git a/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py index 861e915e0..f58cb6b3a 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/function_symbol_collector.py @@ -2,7 +2,6 @@ from typing import Union, Any, List, Optional, Set, Tuple from cc_python_parser.common.file_position import FilePosition -from cc_python_parser.common.parser_tree import ParserTree from cc_python_parser.common.utils import create_range_from_ast_node from cc_python_parser.function_data import FunctionDeclaration from cc_python_parser.scope import FunctionScope @@ -18,9 +17,9 @@ class TemporaryScope(FunctionScope): class FunctionSymbolCollector(SymbolCollector, IFunctionSymbolCollector): - def __init__(self, symbol_collector: SymbolCollector, tree: Union[ast.FunctionDef, ast.AsyncFunctionDef], + def __init__(self, symbol_collector: SymbolCollector, root: Union[ast.FunctionDef, ast.AsyncFunctionDef], arguments: List[Tuple[DeclarationType, Optional[str]]]): - SymbolCollector.__init__(self, ParserTree(tree), symbol_collector.current_file, + SymbolCollector.__init__(self, root, symbol_collector.current_file, symbol_collector.preprocessed_file, symbol_collector.import_finder, symbol_collector.scope_manager.persistence, @@ -28,7 +27,8 @@ def __init__(self, symbol_collector: SymbolCollector, tree: Union[ast.FunctionDe IFunctionSymbolCollector.__init__(self) self.scope_manager = symbol_collector.scope_manager self.type_deduction = TypeDeduction(self, self.scope_manager, self.preprocessed_file, - self.function_symbol_collector_factory) + self.function_symbol_collector_factory, + symbol_collector.scope_manager.persistence) self.imported_declaration_scope_map = symbol_collector.imported_declaration_scope_map self.arguments = arguments diff --git a/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py index 9dab36bf6..6f05fef11 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/import_preprocessor.py @@ -101,7 +101,7 @@ def convert_ast_import_to_import(node: ast.Import, is_local: bool = False) -> Li for module in node.names: module_parts = module.name.split('.') module_name = module_parts[-1] - module_parts.remove(module_name) + del module_parts[-1] r = create_range_from_ast_node(node) imports.append(ImportTable.create_import(module_parts, module_name, module.asname, [], r, is_local)) return imports @@ -112,8 +112,7 @@ def convert_ast_import_from_to_import(node: ast.ImportFrom, is_local: bool = Fal is_module = True try: if node.level > 0 and node.module is None: - a = util.find_spec(node.names[0].name) - if a is None: + if util.find_spec(node.names[0].name) is None: pass # print() elif (node.level == 0 or node.module is not None) and \ util.find_spec(node.module + '.' + node.names[0].name) is None: diff --git a/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py index 5348bbd83..17f601d58 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/member_access_collector.py @@ -2,6 +2,8 @@ import sys from typing import Optional, List, Any, Union +from cc_python_parser.persistence.persistence import ModelPersistence + class MemberAccessCollector(ast.NodeVisitor): class MemberData: @@ -64,10 +66,11 @@ def __init__(self, node): super().__init__('') self.container = node - def __init__(self, member: Union[ast.Call, ast.Attribute]): + def __init__(self, member: Union[ast.Call, ast.Attribute], persistence: ModelPersistence): self.call_list: List[MemberAccessCollector.MemberData] = [] self.arguments = [] self.last = False + self.persistence: ModelPersistence = persistence self.visit(member) def generic_visit(self, node: ast.AST): @@ -75,7 +78,7 @@ def generic_visit(self, node: ast.AST): return # await and starred must be skipped, and process the callable if not isinstance(node, (ast.Await, ast.Starred)): - print('Unhandled node type: ' + str(type(node))) + self.persistence.log_warning(f'Unhandled node type: {str(type(node))}') ast.NodeVisitor.generic_visit(self, node) def visit_Call(self, node: ast.Call) -> Any: diff --git a/plugins/python/parser/src/scripts/cc_python_parser/parser.py b/plugins/python/parser/src/scripts/cc_python_parser/parser.py index 81a3c219b..d79e66b73 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/parser.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/parser.py @@ -4,7 +4,6 @@ from pathlib import PurePath, Path from typing import List, Optional, Union, Set, Tuple -from cc_python_parser.common.parser_tree import ParserTree from cc_python_parser.common.utils import process_file_content from cc_python_parser.file_info import FileInfo, ProcessStatus from cc_python_parser.function_symbol_collector import FunctionSymbolCollector @@ -37,8 +36,6 @@ def __init__(self, directories: List[str], persistence: ModelPersistence, exceptions = DefaultParseException() self.exceptions: ParseException = exceptions self.files: List[FileInfo] = [] - self.other_module_files: List[FileInfo] = [] - self.parsing_started_files = [] self.scope_managers = [] self.collect_files() @@ -59,7 +56,7 @@ def process_directory(self, directory: PurePath): continue file_path = file_path.resolve() if file_path.suffix == '.py': - self.files.append(FileInfo(file_path)) + self.files.append(FileInfo(file_path, self.persistence)) def parse(self) -> None: metrics.start_parsing() @@ -80,22 +77,22 @@ def parse_file(self, file_info: FileInfo) -> None: if file_info.path is None: return - tree = None + root = None def handle_file_content(c, line_num): - nonlocal tree + nonlocal root try: - tree = ast.parse(c) + root = ast.parse(c) metrics.add_line_count(line_num) metrics.add_file_count() except SyntaxError as e: - print(f"Syntax error in file {e.filename} at (line - {e.lineno}, column - {e.offset}): {e.text}") + self.persistence.log_error(f"Syntax error in file {e.filename} at (line - {e.lineno}, column - {e.offset}): {e.text}") - process_file_content(file_info.path, handle_file_content) + process_file_content(file_info.path, handle_file_content, self.persistence) - if tree is None: + if root is None: return - file_info.preprocess_file(tree) + file_info.preprocess_file(root) self.handle_modules_outside_of_project(file_info) for dependency in file_info.preprocessed_file.import_table.get_dependencies(): dependency_file_info = [x for x in self.files if x.path == dependency.location] @@ -110,7 +107,7 @@ def handle_file_content(c, line_num): debug(f"\nFILE: {file_info.get_file_name()}\n=========================================") else: assert False, 'Multiple file occurrence: ' + dependency.get_file() - sc = SymbolCollector(ParserTree(tree), file_info.path, file_info.preprocessed_file, + sc = SymbolCollector(root, file_info.path, file_info.preprocessed_file, self, self.persistence, self) # print(current_file) sc.collect_symbols() @@ -127,8 +124,7 @@ def handle_modules_outside_of_project(self, file_info: FileInfo) -> None: if not any(PurePath(mm) in m.parents for mm in self.directories) and \ m not in self.other_modules: self.other_modules.add(m) - new_file_info = FileInfo(m) - self.other_module_files.append(new_file_info) + new_file_info = FileInfo(m, self.persistence) self.parse_file(new_file_info) def get_file_by_location(self, location: PurePath) -> Optional[FileInfo]: diff --git a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py index 298e670db..dab5eab44 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/persistence/persistence.py @@ -19,6 +19,10 @@ def __del__(self): def check_c_persistence(self): if self.c_persistence is None: return + assert hasattr(self.c_persistence, 'log_info') + assert hasattr(self.c_persistence, 'log_warning') + assert hasattr(self.c_persistence, 'log_error') + assert hasattr(self.c_persistence, 'log_debug') assert hasattr(self.c_persistence, 'persist_file') assert hasattr(self.c_persistence, 'persist_variable') assert hasattr(self.c_persistence, 'persist_function') @@ -26,48 +30,72 @@ def check_c_persistence(self): assert hasattr(self.c_persistence, 'persist_class') assert hasattr(self.c_persistence, 'persist_import') + def log_info(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_info(message) + else: + self.log.write(message) + + def log_warning(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_warning(message) + else: + self.log.write(message) + + def log_error(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_error(message) + else: + self.log.write(message) + + def log_debug(self, message: str): + if self.c_persistence is not None: + self.c_persistence.log_debug(message) + else: + self.log.write(message) + def persist_file(self, file: FileDTO) -> None: if self.c_persistence is not None: - self.c_persistence.print(f"Persist file: {file.path}") + self.log_debug(f"Persist file: {file.path}") self.c_persistence.persist_file(file) else: self.log.write(f"Persist file: {file.path}\n") def persist_variable(self, declaration: VariableDeclarationDTO) -> None: if self.c_persistence is not None: - self.c_persistence.print(f"Persist var: {declaration.qualified_name}") + self.log_debug(f"Persist var: {declaration.qualified_name}") self.c_persistence.persist_variable(declaration) else: self.log.write(f"Persist var: {declaration.qualified_name}\n") def persist_function(self, declaration: FunctionDeclarationDTO) -> None: if self.c_persistence is not None: - self.c_persistence.print(f"Persist func: {declaration.qualified_name}") + self.log_debug(f"Persist func: {declaration.qualified_name}") self.c_persistence.persist_function(declaration) else: self.log.write(f"Persist func: {declaration.qualified_name}\n") def persist_preprocessed_class(self, declaration: ClassDeclarationDTO) -> None: if self.c_persistence is not None: - self.c_persistence.print(f"Persist preprocessed class: {declaration.qualified_name}") + self.log_debug(f"Persist preprocessed class: {declaration.qualified_name}") self.c_persistence.persist_preprocessed_class(declaration) else: self.log.write(f"Persist preprocessed class: {declaration.qualified_name}\n") def persist_class(self, declaration: ClassDeclarationDTO) -> None: if self.c_persistence is not None: - self.c_persistence.print(f"Persist class: {declaration.qualified_name}") + self.log_debug(f"Persist class: {declaration.qualified_name}") self.c_persistence.persist_class(declaration) else: self.log.write(f"Persist class: {declaration.qualified_name}\n") def persist_import(self, imports: ImportDTO) -> None: if self.c_persistence is not None: - self.c_persistence.print("Persist import") + self.log_debug("Persist import") for i in imports.imported_modules: - self.c_persistence.print(f"Persist imported module: {i.qualified_name}") + self.log_debug(f"Persist imported module: {i.qualified_name}") for i in imports.imported_symbols: - self.c_persistence.print(f"Persist imported symbol: {i.qualified_name}") + self.log_debug(f"Persist imported symbol: {i.qualified_name}") self.c_persistence.persist_import(imports) else: for i in imports.imported_modules: diff --git a/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py b/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py index e9b18a037..d2ed1e086 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/python_parser.py @@ -1,25 +1,12 @@ -import os -from pathlib import PurePath - -from cc_python_parser.parse_exception import ParseException from cc_python_parser.parser import Parser +from cc_python_parser.skip_file import read_skip_file from cc_python_parser.persistence.persistence import init_persistence, ModelPersistence def parse(source_root: str, persistence): - init_persistence(persistence) - - def directory_exception(path: PurePath) -> bool: - directory = os.path.basename(os.path.normpath(str(path))) - return directory.startswith('.') or directory == 'venv' - - def file_exception(path: PurePath) -> bool: - return False - - exception = ParseException(directory_exception, file_exception) - p = Parser([source_root], ModelPersistence(persistence), exception) + model_persistence = ModelPersistence(persistence) + p = Parser([source_root], model_persistence, read_skip_file(model_persistence)) p.parse() - pass if __name__ == '__main__': diff --git a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py index 041445bd7..c5cfd7cc4 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/scope_manager.py @@ -476,13 +476,9 @@ def persist_current_scope(self): def get_qualified_scope_name_from_current_scope(self, scopes: Optional[List[Scope]] = None) -> str: qualified_name_parts = [] if scopes is None: - for scope in reversed(self.scopes): - if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope - qualified_name_parts.append(scope.name) + qualified_name_parts.extend(self.get_qualified_scope_name_from_scopes(self.scopes)) else: - for scope in reversed(scopes): - if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope - qualified_name_parts.append(scope.name) + qualified_name_parts.extend(self.get_qualified_scope_name_from_scopes(scopes)) file_name = self.current_file.name assert file_name[-3:] == '.py' qualified_name_parts.append(file_name[:-3]) @@ -497,5 +493,12 @@ def get_qualified_scope_name_from_current_scope(self, scopes: Optional[List[Scop break return '.'.join(reversed(qualified_name_parts)) + def get_qualified_scope_name_from_scopes(self, scopes: List[Scope]): + qualified_name_parts = [] + for scope in reversed(scopes): + if isinstance(scope, LifetimeScope) and not isinstance(scope, GlobalScope): # TODO: ExceptionScope + qualified_name_parts.append(scope.name) + return qualified_name_parts + def get_qualified_name_from_current_scope(self, declaration_name: str, line_num: int) -> str: return self.get_qualified_scope_name_from_current_scope() + '.' + declaration_name + ':' + str(line_num) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/skip_file.py b/plugins/python/parser/src/scripts/cc_python_parser/skip_file.py new file mode 100644 index 000000000..2d1449f9d --- /dev/null +++ b/plugins/python/parser/src/scripts/cc_python_parser/skip_file.py @@ -0,0 +1,46 @@ +import json +import re +import os +from json import JSONDecodeError +from typing import Final +from pathlib import Path + +from cc_python_parser.persistence.persistence import ModelPersistence +from cc_python_parser.parse_exception import ParseException + +SKIP_FILE_NAME: Final[str] = "skip.json" + + +def read_skip_file(persistence: ModelPersistence) -> ParseException: + """ + Reads regexes to skip files and directories from 'skip.json' while parsing, if it exists: + { + "files": [], + "directories": [ + "^[.]", + "^venv$" + ] + } + """ + + ff = lambda p: False + dd = lambda p: False + + split_path = __file__.split(os.sep) + del split_path[-1] + split_path[-1] = SKIP_FILE_NAME + path = f"{os.sep}{os.path.join(*split_path)}" + + if Path(path).exists(): + with open(path, 'r') as config_file: + try: + skip = json.load(config_file) + if skip['files']: + ff = lambda p: any(re.search(f, p) is not None for f in skip['files']) + if skip['directories']: + dd = lambda p: any(re.search(d, p) is not None for d in skip['directories']) + except JSONDecodeError as e: + persistence.log_warning(f"Invalid JSON file: {SKIP_FILE_NAME} (line: {e.lineno}, column: {e.colno})") + except Exception as e: + persistence.log_warning(f"Error during {SKIP_FILE_NAME} decoding: {e}") + return ParseException(dd, ff) diff --git a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py index 736092267..e902d705b 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/symbol_collector.py @@ -13,7 +13,6 @@ from cc_python_parser.base_data import Declaration, Usage, ImportedDeclaration from cc_python_parser.class_data import ClassDeclaration, ImportedClassDeclaration from cc_python_parser.class_preprocessor import PreprocessedClass -from cc_python_parser.common.parser_tree import ParserTree from cc_python_parser.import_finder import ImportFinder from cc_python_parser.import_preprocessor import ImportTable from cc_python_parser.logger import logger @@ -37,21 +36,22 @@ class SymbolCollector(ast.NodeVisitor, SymbolFinder, SymbolCollectorBase): - def __init__(self, tree: ParserTree, current_file: PurePath, preprocessed_file: PreprocessedFile, + def __init__(self, root, current_file: PurePath, preprocessed_file: PreprocessedFile, import_finder: ImportFinder, persistence: ModelPersistence, function_symbol_collector_factory: FunctionSymbolCollectorFactory): SymbolCollectorBase.__init__(self, preprocessed_file, import_finder) - self.tree = tree + self.root = root self.current_file = current_file self.scope_manager: ScopeManager = ScopeManager(current_file, import_finder, persistence) + self.persistence: ModelPersistence = persistence self.current_function_declaration: List[FunctionDeclaration] = [] self.current_class_declaration: List[ClassDeclaration] = [] self.function_symbol_collector_factory = function_symbol_collector_factory self.type_deduction = TypeDeduction(self, self.scope_manager, - preprocessed_file, self.function_symbol_collector_factory) + preprocessed_file, self.function_symbol_collector_factory, self.persistence) def collect_symbols(self): - self.visit(self.tree.root.node) + self.visit(self.root) self.post_process() def generic_visit(self, node: ast.AST) -> Any: @@ -437,7 +437,7 @@ def handle_assignment(self, target: ast.AST, value: ast.AST): def handle_single_assignment(self, target: ast.AST, value: ast.AST): mac = None if isinstance(target, (ast.Attribute, ast.Subscript, ast.Starred)): - mac = MemberAccessCollector(target) + mac = MemberAccessCollector(target, self.persistence) name = mac.call_list[0].name elif isinstance(target, ast.Name): name = target.id @@ -882,7 +882,7 @@ def append_variable_usage(self, name: str, node: (ast.Name, ast.Attribute, ast.S def append_function_usage(self, node: ast.Call) -> None: usage = self.create_function_usage(node) - mac = self.member_access_collector_type(node) + mac = self.member_access_collector_type(node, self.persistence) is_method_call = (len(mac.call_list) == 2 and isinstance(mac.call_list[1], MemberAccessCollector.AttributeData) and mac.call_list[1].name == 'self') @@ -922,7 +922,7 @@ def get_declaration(self, node: (ast.Name, ast.Attribute, ast.Call, ast.Subscrip else: return {declaration} - mac = MemberAccessCollector(node) + mac = MemberAccessCollector(node, self.persistence) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and len(mac.call_list) == 1: declaration = self.scope_manager.get_declaration(node.func.id) if declaration is None or isinstance(declaration, PlaceholderType): @@ -936,7 +936,7 @@ def get_declaration(self, node: (ast.Name, ast.Attribute, ast.Call, ast.Subscrip elif isinstance(node, ast.Call) and len(mac.call_list) == 1 and not isinstance(node.func, ast.Subscript): pass # print("NEED FIX: () operator called on lambda, operator or await") - mac = MemberAccessCollector(node) + mac = MemberAccessCollector(node, self.persistence) last = mac.call_list.pop(0) types = self.type_deduction.get_member_access_type(mac) declarations = set() @@ -1106,7 +1106,7 @@ def create_function_usage(self, func: ast.Call) -> Usage: # asdf = '.'.join(n) - mac = MemberAccessCollector(func) + mac = MemberAccessCollector(func, self.persistence) if hasattr(func.func, 'id'): name = func.func.id diff --git a/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py b/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py index fdb029e85..85fc7912e 100644 --- a/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py +++ b/plugins/python/parser/src/scripts/cc_python_parser/type_deduction.py @@ -10,6 +10,7 @@ from cc_python_parser.built_in_types import Boolean, Dictionary, Set, Tuple, String, Integer, Float, Bytes, \ EllipsisType, NoneType, Complex, RangeType, BuiltIn, Type, GenericBuiltInType, NotImplementedType, \ get_built_in_type, GenericType, Generator +from cc_python_parser.persistence.persistence import ModelPersistence from cc_python_parser.class_data import ClassDeclaration from cc_python_parser.common.file_position import FilePosition from cc_python_parser.common.hashable_list import OrderedHashableList @@ -31,13 +32,15 @@ class TypeDeduction: def __init__(self, symbol_collector: SymbolCollectorBase, scope_manager: ScopeManager, preprocessed_file: PreprocessedFile, - function_symbol_collector_factory: FunctionSymbolCollectorFactory): + function_symbol_collector_factory: FunctionSymbolCollectorFactory, + persistence: ModelPersistence): self.scope_manager: ScopeManager = scope_manager self.preprocessed_file: PreprocessedFile = preprocessed_file self.symbol_collector: SymbolCollectorBase = symbol_collector self.function_symbol_collector_factory: FunctionSymbolCollectorFactory = function_symbol_collector_factory self.container_recursion_counter: int = 0 self.container_max_recursion: int = 5 + self.persistence: ModelPersistence = persistence def deduct_type(self, node: ast.AST) -> typing.Set: types = self.get_type(node) @@ -211,7 +214,7 @@ def _(self, node: ast.Call): return built_in_function elif node.func.id == 'TypeVar': return Type() - return self.get_member_access_type(MemberAccessCollector(node), + return self.get_member_access_type(MemberAccessCollector(node, self.persistence), FilePosition(self.scope_manager.current_file, create_range_from_ast_node(node))) @@ -246,7 +249,7 @@ def _(self, node: ast.Constant): @get_type.register def _(self, node: ast.Attribute): - return self.get_member_access_type(MemberAccessCollector(node), + return self.get_member_access_type(MemberAccessCollector(node, self.persistence), FilePosition(self.scope_manager.current_file, create_range_from_ast_node(node))) diff --git a/plugins/python/service/include/service/pythonservice.h b/plugins/python/service/include/service/pythonservice.h index d79bbd5db..3d59fe8f3 100644 --- a/plugins/python/service/include/service/pythonservice.h +++ b/plugins/python/service/include/service/pythonservice.h @@ -145,28 +145,12 @@ class PythonServiceHandler : virtual public LanguageServiceIf CALLER, /*!< Get caller functions. */ -// VIRTUAL_CALL, /*!< A function may be used virtually on a base type object. -// The exact type of the object is based on dynamic information, which can't -// be determined statically. Weak usage returns these possible calls. */ - -// FUNC_PTR_CALL, /*!< Functions can be assigned to function pointers which -// can be invoked later. This option returns these invocations. */ - PARAMETER, /*!< This option returns the parameters of a function. */ LOCAL_VAR, /*!< This option returns the local variables of a function. */ RETURN_TYPE, /*!< This option returns the return type of a function. */ -// OVERRIDE, /*!< This option returns the functions which the given function -// overrides. */ -// -// OVERRIDDEN_BY, /*!< This option returns the overrides of a function. */ -// -// READ, /*!< This option returns the places where a variable is read. */ -// -// WRITE, /*!< This option returns the places where a variable is written. */ - TYPE, /*!< This option returns the type of a variable. */ INHERIT_FROM, /*!< Types from which the queried type inherits. */ From 57377151fc0be00288f6edbff2ab4c027682ee7d Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 15 May 2022 18:42:57 +0200 Subject: [PATCH 38/39] Store AstNodes in set Introduce Interpret BuildAction type --- model/include/model/buildaction.h | 1 + .../include/pythonparser/pythonpersistence.h | 40 +- .../python/parser/src/pythonpersistence.cpp | 436 ++++++++---------- 3 files changed, 230 insertions(+), 247 deletions(-) diff --git a/model/include/model/buildaction.h b/model/include/model/buildaction.h index 382ca817a..8b086ae4c 100644 --- a/model/include/model/buildaction.h +++ b/model/include/model/buildaction.h @@ -25,6 +25,7 @@ struct BuildAction { Compile, Link, + Interpret, Other }; diff --git a/plugins/python/parser/include/pythonparser/pythonpersistence.h b/plugins/python/parser/include/pythonparser/pythonpersistence.h index 5460c6eff..adff61a87 100644 --- a/plugins/python/parser/include/pythonparser/pythonpersistence.h +++ b/plugins/python/parser/include/pythonparser/pythonpersistence.h @@ -43,6 +43,22 @@ namespace parser { class PythonPersistence { private: + struct AstNodeEquality { + public: + bool operator()(const model::PythonAstNodePtr& first, const model::PythonAstNodePtr& second) const { + return first->id == second->id; + } + }; + + struct AstNodeHash { + public: + size_t operator()(const model::PythonAstNodePtr& node) const { + return node->id; + } + }; + + using AstNodeSet = std::unordered_set; + ParserContext &ctx; public: @@ -83,18 +99,18 @@ class PythonPersistence { bool isAstNodePersisted(const std::vector &nodes, const model::PythonAstNodePtr &node) const; - std::vector _astNodes; - std::vector _variables; - std::map > _variableUsages; - std::vector _functions; - std::map > _functionUsages; - std::vector _classes; - std::map > _classUsages; - std::vector _members; - std::vector _inheritance; - std::vector _imports; - std::vector _documentations; - std::vector _types; + AstNodeSet _astNodes; + std::vector _variables; + std::map _variableUsages; + std::vector _functions; + std::map _functionUsages; + std::vector _classes; + std::map _classUsages; + std::vector _members; + std::vector _inheritance; + std::vector _imports; + std::vector _documentations; + std::vector _types; }; typedef boost::shared_ptr PythonPersistencePtr; diff --git a/plugins/python/parser/src/pythonpersistence.cpp b/plugins/python/parser/src/pythonpersistence.cpp index e81968127..525630557 100644 --- a/plugins/python/parser/src/pythonpersistence.cpp +++ b/plugins/python/parser/src/pythonpersistence.cpp @@ -1,13 +1,18 @@ #include -#include - - namespace cc { namespace parser { +using boost::python::object; +using boost::python::len; +using boost::python::tuple; +using boost::python::list; +using boost::python::dict; + +template +using extract = boost::python::extract; void print(const std::string& s) { @@ -30,18 +35,22 @@ PythonPersistence::~PythonPersistence() ctx.srcMgr.persistFiles(); (util::OdbTransaction(ctx.db))([this]{ - util::persistAll(_astNodes, ctx.db); + std::vector v = std::vector(_astNodes.begin(), _astNodes.end()); + util::persistAll(v, ctx.db); for(auto& ast : _variableUsages) { - util::persistAll(ast.second, ctx.db); + std::vector u = std::vector(ast.second.begin(), ast.second.end()); + util::persistAll(u, ctx.db); } for(auto& ast : _functionUsages) { - util::persistAll(ast.second, ctx.db); + std::vector u = std::vector(ast.second.begin(), ast.second.end()); + util::persistAll(u, ctx.db); } for(auto& ast : _classUsages) { - util::persistAll(ast.second, ctx.db); + std::vector u = std::vector(ast.second.begin(), ast.second.end()); + util::persistAll(u, ctx.db); } util::persistAll(_variables, ctx.db); util::persistAll(_functions, ctx.db); @@ -54,44 +63,44 @@ PythonPersistence::~PythonPersistence() }); } -void PythonPersistence::cppprint(boost::python::object o) +void PythonPersistence::cppprint(object o) { - std::string s = boost::python::extract(o); + std::string s = extract(o); LOG(info) << s; } -void PythonPersistence::logInfo(boost::python::object o) +void PythonPersistence::logInfo(object o) { - std::string s = boost::python::extract(o); + std::string s = extract(o); LOG(info) << s; } -void PythonPersistence::logWarning(boost::python::object o) +void PythonPersistence::logWarning(object o) { - std::string s = boost::python::extract(o); + std::string s = extract(o); LOG(warning) << s; } -void PythonPersistence::logError(boost::python::object o) +void PythonPersistence::logError(object o) { - std::string s = boost::python::extract(o); + std::string s = extract(o); LOG(error) << s; } -void PythonPersistence::logDebug(boost::python::object o) +void PythonPersistence::logDebug(object o) { - std::string s = boost::python::extract(o); + std::string s = extract(o); LOG(debug) << s; } -void PythonPersistence::persistFile(boost::python::object pyFile) +void PythonPersistence::persistFile(object pyFile) { try{ model::FilePtr file = nullptr; model::BuildSource buildSource; - boost::python::object path = pyFile.attr("path"); - boost::python::object status = pyFile.attr("parse_status"); + object path = pyFile.attr("path"); + object status = pyFile.attr("parse_status"); if(status.is_none()) { @@ -100,10 +109,10 @@ void PythonPersistence::persistFile(boost::python::object pyFile) { LOG(error) << "Persist file: path is None..."; } else { - file = ctx.srcMgr.getFile(boost::python::extract(path)); + file = ctx.srcMgr.getFile(extract(path)); file->type = "PY"; buildSource.file = file; - switch(boost::python::extract(status)) + switch(extract(status)) { case 0: buildSource.file->parseStatus = model::File::PSNone; @@ -115,12 +124,12 @@ void PythonPersistence::persistFile(boost::python::object pyFile) buildSource.file->parseStatus = model::File::PSFullyParsed; break; default: - LOG(error) << "Persist file unknown status: " << boost::python::extract(status); + LOG(error) << "Persist file unknown status: " << extract(status); } model::BuildActionPtr buildAction(new model::BuildAction); buildAction->command = ""; - buildAction->type = model::BuildAction::Other; + buildAction->type = model::BuildAction::Interpret; try{ util::OdbTransaction transaction{ctx.db}; @@ -146,21 +155,21 @@ void PythonPersistence::persistFile(boost::python::object pyFile) } } -void PythonPersistence::persistVariable(boost::python::object pyVariable) +void PythonPersistence::persistVariable(object pyVariable) { try{ - boost::python::object name = pyVariable.attr("name"); - boost::python::object qualifiedName = pyVariable.attr("qualified_name"); - boost::python::object visibility = pyVariable.attr("visibility"); + object name = pyVariable.attr("name"); + object qualifiedName = pyVariable.attr("qualified_name"); + object visibility = pyVariable.attr("visibility"); boost::optional fileLoc = createFileLoc(pyVariable.attr("file_position")); - boost::python::list types = - boost::python::extract(pyVariable.attr("type")); + list types = + extract(pyVariable.attr("type")); - boost::python::list usages = - boost::python::extract(pyVariable.attr("usages")); + list usages = + extract(pyVariable.attr("usages")); if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || types.is_none() || usages.is_none()) @@ -170,33 +179,32 @@ void PythonPersistence::persistVariable(boost::python::object pyVariable) model::PythonAstNodePtr varAstNode(new model::PythonAstNode); varAstNode->location = fileLoc.get(); - varAstNode->qualifiedName = boost::python::extract(qualifiedName); + varAstNode->qualifiedName = extract(qualifiedName); varAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; varAstNode->astType = model::PythonAstNode::AstType::Declaration; varAstNode->id = model::createIdentifier(*varAstNode); - if(isAstNodePersisted(varAstNode)) + if(!_astNodes.insert(varAstNode).second) { return; } - _astNodes.push_back(varAstNode); model::PythonVariablePtr variable(new model::PythonVariable); variable->astNodeId = varAstNode->id; - variable->name = boost::python::extract(name); - variable->qualifiedName = boost::python::extract(qualifiedName); - variable->visibility = boost::python::extract(visibility); + variable->name = extract(name); + variable->qualifiedName = extract(qualifiedName); + variable->visibility = extract(visibility); variable->id = model::createIdentifier(*variable); _variables.push_back(variable); - for(int i = 0; i(types[i]); + std::string s = extract(types[i]); model::PythonEntityPtr t = - getPythonEntity(boost::python::extract(types[i])); + getPythonEntity(extract(types[i])); if(!t) { continue; @@ -207,7 +215,7 @@ void PythonPersistence::persistVariable(boost::python::object pyVariable) } _variableUsages[variable->id] = {}; - for(int i = 0; i usageFileLoc = createFileLoc(usages[i].attr("file_position")); @@ -217,18 +225,13 @@ void PythonPersistence::persistVariable(boost::python::object pyVariable) } model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->qualifiedName = extract(qualifiedName); usageAstNode->symbolType = model::PythonAstNode::SymbolType::Variable; usageAstNode->astType = model::PythonAstNode::AstType::Usage; usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_variableUsages[variable->id], usageAstNode)) - { - continue; - } - - _variableUsages[variable->id].push_back(usageAstNode); + _variableUsages[variable->id].insert(usageAstNode); } } catch (const odb::object_already_persistent& ex) @@ -243,29 +246,29 @@ void PythonPersistence::persistVariable(boost::python::object pyVariable) } } -void PythonPersistence::persistFunction(boost::python::object pyFunction) +void PythonPersistence::persistFunction(object pyFunction) { try{ - boost::python::object name = pyFunction.attr("name"); - boost::python::object qualifiedName = pyFunction.attr("qualified_name"); - boost::python::object visibility = pyFunction.attr("visibility"); + object name = pyFunction.attr("name"); + object qualifiedName = pyFunction.attr("qualified_name"); + object visibility = pyFunction.attr("visibility"); boost::optional fileLoc = createFileLoc(pyFunction.attr("file_position")); - boost::python::list types = - boost::python::extract(pyFunction.attr("type")); + list types = + extract(pyFunction.attr("type")); - boost::python::list usages = - boost::python::extract(pyFunction.attr("usages")); + list usages = + extract(pyFunction.attr("usages")); - boost::python::object pyDocumentation = pyFunction.attr("documentation"); + object pyDocumentation = pyFunction.attr("documentation"); - boost::python::list params = - boost::python::extract(pyFunction.attr("parameters")); + list params = + extract(pyFunction.attr("parameters")); - boost::python::list locals = - boost::python::extract(pyFunction.attr("locals")); + list locals = + extract(pyFunction.attr("locals")); if(name.is_none() || qualifiedName.is_none() || fileLoc == boost::none || types.is_none() || usages.is_none() || pyDocumentation.is_none() || params.is_none() || locals.is_none()) @@ -275,31 +278,29 @@ void PythonPersistence::persistFunction(boost::python::object pyFunction) model::PythonAstNodePtr funcAstNode(new model::PythonAstNode); funcAstNode->location = fileLoc.get(); - funcAstNode->qualifiedName = boost::python::extract(qualifiedName); + funcAstNode->qualifiedName = extract(qualifiedName); funcAstNode->symbolType = model::PythonAstNode::SymbolType::Function; funcAstNode->astType = model::PythonAstNode::AstType::Declaration; funcAstNode->id = model::createIdentifier(*funcAstNode); - if(isAstNodePersisted(funcAstNode)) + if(!_astNodes.insert(funcAstNode).second) { return; } - _astNodes.push_back(funcAstNode); - model::PythonFunctionPtr function(new model::PythonFunction); function->astNodeId = funcAstNode->id; - function->name = boost::python::extract(name); - function->qualifiedName = boost::python::extract(qualifiedName); - function->visibility = boost::python::extract(visibility); + function->name = extract(name); + function->qualifiedName = extract(qualifiedName); + function->visibility = extract(visibility); function->id = model::createIdentifier(*function); - for(int i = 0; i(params[i])); + getPythonVariable(extract(params[i])); if(!param) { continue; @@ -307,10 +308,10 @@ void PythonPersistence::persistFunction(boost::python::object pyFunction) function->parameters.push_back(param); } - for(int i = 0; i(locals[i])); + getPythonVariable(extract(locals[i])); if(!local) { continue; @@ -321,17 +322,17 @@ void PythonPersistence::persistFunction(boost::python::object pyFunction) _functions.push_back(function); model::PythonDocumentationPtr documentation(new model::PythonDocumentation); - documentation->documentation = boost::python::extract(pyDocumentation); + documentation->documentation = extract(pyDocumentation); documentation->documented = function->id; documentation->documentationKind = model::PythonDocumentation::Function; _documentations.push_back(documentation); - for(int i = 0; i(types[i])); + getPythonEntity(extract(types[i])); if(!t) { continue; @@ -343,7 +344,7 @@ void PythonPersistence::persistFunction(boost::python::object pyFunction) } _functionUsages[function->id] = {}; - for(int i = 0; i usageFileLoc = createFileLoc(usages[i].attr("file_position")); @@ -353,18 +354,13 @@ void PythonPersistence::persistFunction(boost::python::object pyFunction) } model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->qualifiedName = extract(qualifiedName); usageAstNode->symbolType = model::PythonAstNode::SymbolType::Function; usageAstNode->astType = model::PythonAstNode::AstType::Usage; usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_functionUsages[function->id], usageAstNode)) - { - continue; - } - - _functionUsages[function->id].push_back(usageAstNode); + _functionUsages[function->id].insert(usageAstNode); } } catch(std::exception e) { @@ -372,12 +368,12 @@ void PythonPersistence::persistFunction(boost::python::object pyFunction) } } -void PythonPersistence::persistPreprocessedClass(boost::python::object pyClass) +void PythonPersistence::persistPreprocessedClass(object pyClass) { try{ - boost::python::object name = pyClass.attr("name"); - boost::python::object qualifiedName = pyClass.attr("qualified_name"); - boost::python::object visibility = pyClass.attr("visibility"); + object name = pyClass.attr("name"); + object qualifiedName = pyClass.attr("qualified_name"); + object visibility = pyClass.attr("visibility"); boost::optional fileLoc = createFileLoc(pyClass.attr("file_position")); @@ -389,24 +385,22 @@ void PythonPersistence::persistPreprocessedClass(boost::python::object pyClass) model::PythonAstNodePtr classAstNode(new model::PythonAstNode); classAstNode->location = fileLoc.get(); - classAstNode->qualifiedName = boost::python::extract(qualifiedName); + classAstNode->qualifiedName = extract(qualifiedName); classAstNode->symbolType = model::PythonAstNode::SymbolType::Class; classAstNode->astType = model::PythonAstNode::AstType::Declaration; classAstNode->id = model::createIdentifier(*classAstNode); - if(isAstNodePersisted(classAstNode)) + if(!_astNodes.insert(classAstNode).second) { return; } - _astNodes.push_back(classAstNode); - model::PythonClassPtr cl(new model::PythonClass); cl->astNodeId = classAstNode->id; - cl->name = boost::python::extract(name); - cl->qualifiedName = boost::python::extract(qualifiedName); - cl->visibility = boost::python::extract(visibility); + cl->name = extract(name); + cl->qualifiedName = extract(qualifiedName); + cl->visibility = extract(visibility); cl->id = model::createIdentifier(*cl); @@ -418,20 +412,20 @@ void PythonPersistence::persistPreprocessedClass(boost::python::object pyClass) } } -void PythonPersistence::persistClass(boost::python::object pyClass) +void PythonPersistence::persistClass(object pyClass) { try{ - boost::python::object qualifiedName = pyClass.attr("qualified_name"); + object qualifiedName = pyClass.attr("qualified_name"); - boost::python::list usages = - boost::python::extract(pyClass.attr("usages")); + list usages = + extract(pyClass.attr("usages")); - boost::python::object pyDocumentation = pyClass.attr("documentation"); + object pyDocumentation = pyClass.attr("documentation"); - boost::python::list baseClasses = - boost::python::extract(pyClass.attr("base_classes")); + list baseClasses = + extract(pyClass.attr("base_classes")); - boost::python::object members = pyClass.attr("members"); + object members = pyClass.attr("members"); if(qualifiedName.is_none() || usages.is_none() || pyDocumentation.is_none() || baseClasses.is_none() || members.is_none()) @@ -440,7 +434,7 @@ void PythonPersistence::persistClass(boost::python::object pyClass) } model::PythonClassPtr cl = - getPythonClass(boost::python::extract(qualifiedName)); + getPythonClass(extract(qualifiedName)); if (!cl) { @@ -449,14 +443,14 @@ void PythonPersistence::persistClass(boost::python::object pyClass) } model::PythonDocumentationPtr documentation(new model::PythonDocumentation); - documentation->documentation = boost::python::extract(pyDocumentation); + documentation->documentation = extract(pyDocumentation); documentation->documented = cl->id; documentation->documentationKind = model::PythonDocumentation::Class; _documentations.push_back(documentation); _classUsages[cl->id] = {}; - for(int i = 0; i usageFileLoc = createFileLoc(usages[i].attr("file_position")); @@ -466,41 +460,36 @@ void PythonPersistence::persistClass(boost::python::object pyClass) } model::PythonAstNodePtr usageAstNode(new model::PythonAstNode); usageAstNode->location = usageFileLoc.get(); - usageAstNode->qualifiedName = boost::python::extract(qualifiedName); + usageAstNode->qualifiedName = extract(qualifiedName); usageAstNode->symbolType = model::PythonAstNode::SymbolType::Class; usageAstNode->astType = model::PythonAstNode::AstType::Usage; usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_classUsages[cl->id], usageAstNode)) - { - continue; - } - - _classUsages[cl->id].push_back(usageAstNode); + _classUsages[cl->id].insert(usageAstNode); } - for(int i = 0; iderived = cl->id; - std::string baseClassQualifiedName = boost::python::extract(baseClasses[i]); + std::string baseClassQualifiedName = extract(baseClasses[i]); inheritance->base = getPythonEntity(baseClassQualifiedName)->id; _inheritance.push_back(inheritance); } - boost::python::list methods = - boost::python::extract(members.attr("methods")); - boost::python::list staticMethods = - boost::python::extract(members.attr("static_methods")); - boost::python::list attributes = - boost::python::extract(members.attr("attributes")); - boost::python::list staticAttributes = - boost::python::extract(members.attr("static_attributes")); - boost::python::list classes = - boost::python::extract(members.attr("classes")); + list methods = + extract(members.attr("methods")); + list staticMethods = + extract(members.attr("static_methods")); + list attributes = + extract(members.attr("attributes")); + list staticAttributes = + extract(members.attr("static_attributes")); + list classes = + extract(members.attr("classes")); if(methods.is_none() || staticMethods.is_none() || attributes.is_none() || staticAttributes.is_none() || classes.is_none()) @@ -508,10 +497,10 @@ void PythonPersistence::persistClass(boost::python::object pyClass) return; } - for(int i = 0; i(methods[i].attr("qualified_name")); + std::string qualifiedName = extract(methods[i].attr("qualified_name")); model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(!method) { @@ -526,10 +515,10 @@ void PythonPersistence::persistClass(boost::python::object pyClass) _members.push_back(classMember); - boost::python::list _usages = - boost::python::extract(methods[i].attr("usages")); - std::vector& _funcUsages = _functionUsages[method->id]; - for(int j = 0; j(methods[i].attr("usages")); + AstNodeSet& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -548,20 +537,15 @@ void PythonPersistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_funcUsages, usageAstNode)) - { - continue; - } - - _funcUsages.push_back(usageAstNode); + _funcUsages.insert(usageAstNode); } } - for(int i = 0; i(staticMethods[i].attr("qualified_name")); + extract(staticMethods[i].attr("qualified_name")); model::PythonEntityPtr method = getPythonEntity(qualifiedName); if(!method) { @@ -576,10 +560,10 @@ void PythonPersistence::persistClass(boost::python::object pyClass) _members.push_back(classMember); - boost::python::list _usages = - boost::python::extract(staticMethods[i].attr("usages")); - std::vector& _funcUsages = _functionUsages[method->id]; - for(int j = 0; j(staticMethods[i].attr("usages")); + AstNodeSet& _funcUsages = _functionUsages[method->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -599,20 +583,15 @@ void PythonPersistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_funcUsages, usageAstNode)) - { - continue; - } - - _funcUsages.push_back(usageAstNode); + _funcUsages.insert(usageAstNode); } } - for(int i = 0; i(attributes[i].attr("qualified_name")); + extract(attributes[i].attr("qualified_name")); if (qualifiedName.empty()) { continue; // TODO: import symbol in class @@ -631,10 +610,10 @@ void PythonPersistence::persistClass(boost::python::object pyClass) _members.push_back(classMember); - boost::python::list _usages = - boost::python::extract(attributes[i].attr("usages")); - std::vector& _varUsages = _variableUsages[attr->id]; - for(int j = 0; j(attributes[i].attr("usages")); + AstNodeSet& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -653,20 +632,15 @@ void PythonPersistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_varUsages, usageAstNode)) - { - continue; - } - - _varUsages.push_back(usageAstNode); + _varUsages.insert(usageAstNode); } } - for(int i = 0; i(staticAttributes[i].attr("qualified_name")); + extract(staticAttributes[i].attr("qualified_name")); model::PythonEntityPtr attr = getPythonEntity(qualifiedName); if(!attr) { @@ -681,10 +655,10 @@ void PythonPersistence::persistClass(boost::python::object pyClass) _members.push_back(classMember); - boost::python::list _usages = - boost::python::extract(staticAttributes[i].attr("usages")); - std::vector& _varUsages = _variableUsages[attr->id]; - for(int j = 0; j(staticAttributes[i].attr("usages")); + AstNodeSet& _varUsages = _variableUsages[attr->id]; + for(int j = 0; j _fl = createFileLoc(_usages[j].attr("file_position")); if(_fl == boost::none || @@ -703,19 +677,14 @@ void PythonPersistence::persistClass(boost::python::object pyClass) usageAstNode->id = model::createIdentifier(*usageAstNode); - if(isAstNodePersisted(_varUsages, usageAstNode)) - { - continue; - } - - _varUsages.push_back(usageAstNode); + _varUsages.insert(usageAstNode); } } - for(int i = 0; i(classes[i].attr("qualified_name")); + std::string qualifiedName = extract(classes[i].attr("qualified_name")); model::PythonEntityPtr inner = getPythonEntity(qualifiedName); if(!inner) { @@ -736,31 +705,31 @@ void PythonPersistence::persistClass(boost::python::object pyClass) } } -void PythonPersistence::persistImport(boost::python::object pyImport) +void PythonPersistence::persistImport(object pyImport) { try { model::FilePtr file = - ctx.srcMgr.getFile(boost::python::extract(pyImport.attr("importer"))); + ctx.srcMgr.getFile(extract(pyImport.attr("importer"))); - boost::python::list importedModules = - boost::python::extract(pyImport.attr("imported_modules")); + list importedModules = + extract(pyImport.attr("imported_modules")); - boost::python::dict importedSymbols = - boost::python::extract(pyImport.attr("imported_symbols")); + dict importedSymbols = + extract(pyImport.attr("imported_symbols")); if (!file || importedModules.is_none() || importedSymbols.is_none()) { return; } - for (int i = 0; i < boost::python::len(importedModules); ++i) + for (int i = 0; i < len(importedModules); ++i) { - boost::python::object importData = importedModules[i]; + object importData = importedModules[i]; model::FilePtr moduleFile = - ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); + ctx.srcMgr.getFile(extract(importData.attr("imported"))); boost::optional fileLoc = createFileLoc(importData.attr("position")); - std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); + std::string qualifiedName = extract(importData.attr("qualified_name")); if (!moduleFile || fileLoc == boost::none) { @@ -775,7 +744,7 @@ void PythonPersistence::persistImport(boost::python::object pyImport) moduleAstNode->id = model::createIdentifier(*moduleAstNode); - if(isAstNodePersisted(moduleAstNode)) + if(!_astNodes.insert(moduleAstNode).second) { continue; } @@ -785,21 +754,20 @@ void PythonPersistence::persistImport(boost::python::object pyImport) moduleImport->importer = file; moduleImport->imported = moduleFile; - _astNodes.push_back(moduleAstNode); _imports.push_back(moduleImport); } - boost::python::list importDict = importedSymbols.items(); - for (int i = 0; i < boost::python::len(importDict); ++i) + list importDict = importedSymbols.items(); + for (int i = 0; i < len(importDict); ++i) { - boost::python::tuple import = boost::python::extract(importDict[i]); + tuple import = extract(importDict[i]); - boost::python::object importData = import[0]; + object importData = import[0]; model::FilePtr moduleFile = - ctx.srcMgr.getFile(boost::python::extract(importData.attr("imported"))); + ctx.srcMgr.getFile(extract(importData.attr("imported"))); boost::optional fileLoc = createFileLoc(importData.attr("position")); - std::string qualifiedName = boost::python::extract(importData.attr("qualified_name")); + std::string qualifiedName = extract(importData.attr("qualified_name")); if (!moduleFile || fileLoc == boost::none) { @@ -814,19 +782,19 @@ void PythonPersistence::persistImport(boost::python::object pyImport) moduleAstNode->id = model::createIdentifier(*moduleAstNode); - if(isAstNodePersisted(moduleAstNode)) + if(!_astNodes.insert(moduleAstNode).second) { continue; } - for (int j = 0; j < boost::python::len(import[1]); ++j) + for (int j = 0; j < len(import[1]); ++j) { model::PythonImportPtr moduleImport(new model::PythonImport); moduleImport->astNodeId = moduleAstNode->id; moduleImport->importer = file; moduleImport->imported = moduleFile; auto symb = - getPythonEntity(boost::python::extract(import[1][j])); + getPythonEntity(extract(import[1][j])); if(!symb) { continue; @@ -835,8 +803,6 @@ void PythonPersistence::persistImport(boost::python::object pyImport) _imports.push_back(moduleImport); } - - _astNodes.push_back(moduleAstNode); } } catch(std::exception e) { @@ -844,23 +810,23 @@ void PythonPersistence::persistImport(boost::python::object pyImport) } } -boost::optional PythonPersistence::createFileLoc(boost::python::object filePosition) +boost::optional PythonPersistence::createFileLoc(object filePosition) { if (filePosition.is_none()) { return boost::none; } - boost::python::object filePath = filePosition.attr("file"); - boost::python::object pyRange = filePosition.attr("range"); + object filePath = filePosition.attr("file"); + object pyRange = filePosition.attr("range"); if (filePath.is_none() || pyRange.is_none()) { return boost::none; } - boost::python::object pyStartPosition = pyRange.attr("start_position"); - boost::python::object pyEndPosition = pyRange.attr("end_position"); + object pyStartPosition = pyRange.attr("start_position"); + object pyEndPosition = pyRange.attr("end_position"); if (pyStartPosition.is_none() || pyEndPosition.is_none()) { @@ -869,12 +835,12 @@ boost::optional PythonPersistence::createFileLoc(boost::python:: model::FileLoc fileLoc; - fileLoc.file = ctx.srcMgr.getFile(boost::python::extract(filePath)); + fileLoc.file = ctx.srcMgr.getFile(extract(filePath)); - model::Position startPosition(boost::python::extract(pyStartPosition.attr("line")), - boost::python::extract(pyStartPosition.attr("column"))); - model::Position endPosition(boost::python::extract(pyEndPosition.attr("line")), - boost::python::extract(pyEndPosition.attr("column"))); + model::Position startPosition(extract(pyStartPosition.attr("line")), + extract(pyStartPosition.attr("column"))); + model::Position endPosition(extract(pyEndPosition.attr("line")), + extract(pyEndPosition.attr("column"))); fileLoc.range = model::Range(startPosition, endPosition); @@ -961,33 +927,33 @@ model::PythonClassPtr PythonPersistence::getPythonClass(const std::string& quali return nullptr; } -bool PythonPersistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const -{ - for(auto it : _astNodes) - { - if(*it == *node) - { - return true; - } - } - - return false; -} - -bool PythonPersistence::isAstNodePersisted( - const std::vector& nodes, - const model::PythonAstNodePtr& node) const -{ - for(auto it : nodes) - { - if(*it == *node) - { - return true; - } - } - - return false; -} +//bool PythonPersistence::isAstNodePersisted(const model::PythonAstNodePtr& node) const +//{ +// for(const auto &it : _astNodes) +// { +// if(*it == *node) +// { +// return true; +// } +// } +// +// return false; +//} + +//bool PythonPersistence::isAstNodePersisted( +// const std::vector& nodes, +// const model::PythonAstNodePtr& node) const +//{ +// for(const auto &it : nodes) +// { +// if(*it == *node) +// { +// return true; +// } +// } +// +// return false; +//} } } \ No newline at end of file From 27d430319c28b38ef50b364cf360111958e7064d Mon Sep 17 00:00:00 2001 From: rmfcnb Date: Sun, 15 May 2022 20:07:24 +0200 Subject: [PATCH 39/39] Extend dependencies with python --- doc/deps.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/deps.md b/doc/deps.md index 1fcddb86a..54f95cfe0 100644 --- a/doc/deps.md +++ b/doc/deps.md @@ -40,6 +40,7 @@ be installed from the official repository of the given Linux distribution. - **`libgtest-dev`**: For testing CodeCompass. ***See [Known issues](#known-issues)!*** - **`libldap2-dev`**: For LDAP authentication. +- **`python3.8`**: For python parsing. ## Quick guide @@ -52,7 +53,7 @@ known issues. sudo apt install git cmake make g++ gcc-7-plugin-dev libboost-all-dev \ llvm-10-dev clang-10 libclang-10-dev \ default-jdk libssl1.0-dev libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen \ - libldap2-dev libgtest-dev npm + libldap2-dev libgtest-dev npm python3.8 ``` #### Ubuntu 20.04 ("Focal Fossa") LTS