Support a Reader class for expanding file types (wjohnson#33)

* Completing rewrite of functionality and tests for more OO design. * Cleaning up table and column lineage test naming conventions * Added better docstrings and light flake8 cleanup. * Updating samples to reflect new classes * Updating docs and README
MPRS-Labs · Sep 9, 2020 · 024e728 · 024e728
1 parent ac1a2bd
commit 024e728
Show file tree

Hide file tree

Showing 31 changed files with 1,562 additions and 1,244 deletions.
diff --git a/README.md b/README.md
@@ -22,14 +22,18 @@ The package currently supports:
 
 ### Build and Install from Source
 
+Create a wheel distribution file and install it in your environment.
+
 ```
+python -m pip install wheel
 python setup.py bdist_wheel
-python -m pip install ./dist/pyapacheatlas-0.0.2-py3-none-any.whl
+python -m pip install ./dist/pyapacheatlas-0.0b10-py3-none-any.whl
 ```
 
 ### Create a Client Connection
 
-Provides connectivity to your Atlas / Data Catalog service. Supports getting and uploading entities and type defs.
+Provides connectivity to your Atlas / Data Catalog service. 
+Supports getting and uploading entities and type defs.
 
 ```
 from pyapacheatlas.auth import ServicePrincipalAuthentication
@@ -41,6 +45,9 @@ auth = ServicePrincipalAuthentication(
     client_secret = ""
 )
 
+# Azure Data Catalog Endpoints are:
+# https://{your_catalog_name}.catalog.babylon.azure.com/api/atlas/v2
+
 client = AtlasClient(
     endpoint_url = "https://MYENDPOINT/api/atlas/v2",
     auth = auth
@@ -77,31 +84,26 @@ upload_results = client.upload_entities([ae.to_json()])
 Read from a standardized excel template to create table, column, table process, and column lineage entities.  Follows / Requires the hive bridge style of column lineages.
 
 ```
-from pyapacheatlas.core import GuidTracker, TypeCategory
+from pyapacheatlas.core import TypeCategory
 from pyapacheatlas.scaffolding import column_lineage_scaffold
-from pyapacheatlas.scaffolding.templates import excel_template
-from pyapacheatlas import from_excel
-from pyapacheatlas.readers.excel import ExcelConfiguration
+from pyapacheatlas.readers import ExcelConfiguration, ExcelReader
 
 file_path = "./atlas_excel_template.xlsx"
 # Create the Excel Template
-excel_template(file_path)
+ExcelReader.make_template(file_path)
 
 # Populate the excel file manually!
 
 # Generate the base atlas type defs
 all_type_defs = client.get_typedefs(TypeCategory.ENTITY)
 
 # Create objects for 
-guid_tracker = GuidTracker()
-excel_config = ExcelConfiguration()
+ec = ExcelConfiguration()
+excel_reader = ExcelReader(ec)
 # Read from excel file and convert to 
-entities = from_excel(file_path, excel_config, guid_tracker)
-
-# Prepare a batch by converting everything to json
-batch = {"entities":[e.to_json() for e in entities]}
+entities = excel_reader.parse_lineage(file_path, all_type_defs)
 
-upload_results = client.upload_entities(batch)
+upload_results = client.upload_entities(entities)
 
 print(json.dumps(upload,results,indent=1))
 ```

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -16,7 +16,6 @@ Welcome to PyApacheAtlas's documentation!
    pyapacheatlas.core.rst
    pyapacheatlas.readers.rst
    pyapacheatlas.scaffolding.rst
-   pyapacheatlas.scaffolding.templates.rst
 
 Indices and tables
 ==================

diff --git a/docs/source/pyapacheatlas.readers.core.rst b/docs/source/pyapacheatlas.readers.core.rst
diff --git a/docs/source/pyapacheatlas.readers.rst b/docs/source/pyapacheatlas.readers.rst
@@ -1,16 +1,24 @@
 pyapacheatlas.readers package
 =============================
 
-Subpackages
------------
+Submodules
+----------
 
-.. toctree::
-   :maxdepth: 4
+pyapacheatlas.readers.reader module
+-----------------------------------
 
-   pyapacheatlas.readers.core
+.. automodule:: pyapacheatlas.readers.reader
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
-Submodules
-----------
+pyapacheatlas.readers.lineagemixin module
+-----------------------------------------
+
+.. automodule:: pyapacheatlas.readers.lineagemixin
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 pyapacheatlas.readers.excel module
 ----------------------------------

diff --git a/docs/source/pyapacheatlas.scaffolding.rst b/docs/source/pyapacheatlas.scaffolding.rst
@@ -1,14 +1,6 @@
 pyapacheatlas.scaffolding package
 =================================
 
-Subpackages
------------
-
-.. toctree::
-   :maxdepth: 4
-
-   pyapacheatlas.scaffolding.templates
-
 Submodules
 ----------
 

diff --git a/docs/source/pyapacheatlas.scaffolding.templates.rst b/docs/source/pyapacheatlas.scaffolding.templates.rst
diff --git a/pyapacheatlas/__init__.py b/pyapacheatlas/__init__.py
@@ -1,3 +1 @@
-from .readers import from_excel
-
-__version__ = "0.0b9"
+__version__ = "0.0b10"
diff --git a/pyapacheatlas/core/__init__.py b/pyapacheatlas/core/__init__.py
@@ -1,8 +1,8 @@
 from .client import AtlasClient
-from .entity import AtlasEntity
-from .entity import AtlasProcess
+from .entity import AtlasEntity, AtlasProcess
 from .typedef import (
+    AtlasAttributeDef,
     EntityTypeDef,
+    RelationshipTypeDef,
     TypeCategory
 )
-from .util import GuidTracker
diff --git a/pyapacheatlas/readers/__init__.py b/pyapacheatlas/readers/__init__.py
@@ -1 +1 @@
-from .excel import excel_typeDefs, from_excel
+from .excel import ExcelConfiguration, ExcelReader
diff --git a/pyapacheatlas/readers/core/__init__.py b/pyapacheatlas/readers/core/__init__.py
diff --git a/pyapacheatlas/readers/core/bulkEntities.py b/pyapacheatlas/readers/core/bulkEntities.py
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		from .excel import excel_typeDefs, from_excel
		from .excel import ExcelConfiguration, ExcelReader