From c06a556f9cfcb8115e1737605254b9f9720541eb Mon Sep 17 00:00:00 2001 From: ynaim94-harrys Date: Thu, 22 Sep 2022 10:06:40 -0400 Subject: [PATCH 1/6] Add export file topics stream --- tap_gladly/schemas/export_topics.json | 18 ++++++++++++++++++ tap_gladly/streams.py | 17 +++++++++++++++++ tap_gladly/tap.py | 2 ++ 3 files changed, 37 insertions(+) create mode 100644 tap_gladly/schemas/export_topics.json diff --git a/tap_gladly/schemas/export_topics.json b/tap_gladly/schemas/export_topics.json new file mode 100644 index 0000000..f12c778 --- /dev/null +++ b/tap_gladly/schemas/export_topics.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "disabled": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "parentId": { + "type": "string" + } + } +} diff --git a/tap_gladly/streams.py b/tap_gladly/streams.py index 42c804d..ca468ab 100644 --- a/tap_gladly/streams.py +++ b/tap_gladly/streams.py @@ -41,6 +41,23 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: return {"job_id": record["id"]} +class ExportFileTopicsStream(gladlyStream): + """Abstract class, export conversation items stream.""" + + name = "topics" + path = "/export/jobs/{job_id}/files/topics.jsonl" + primary_keys = ["id"] + replication_key = None + parent_stream_type = ExportJobsStream + ignore_parent_replication_key = True + schema_filepath = SCHEMAS_DIR / "export_topics.json" + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result records.""" + for line in response.iter_lines(): + yield from extract_jsonpath(self.records_jsonpath, input=json.loads(line)) + + class ExportFileConversationItemsStream(gladlyStream, abc.ABC): """Abstract class, export conversation items stream.""" diff --git a/tap_gladly/tap.py b/tap_gladly/tap.py index 8e8f452..82aedf5 100644 --- a/tap_gladly/tap.py +++ b/tap_gladly/tap.py @@ -14,6 +14,7 @@ ExportFileConversationItemsSms, ExportFileConversationItemsTopicChange, ExportFileConversationItemsVoiceMail, + ExportFileTopicsStream, ExportJobsStream, ) @@ -26,6 +27,7 @@ ExportFileConversationItemsConversationStatusChange, ExportFileConversationItemsPhoneCall, ExportFileConversationItemsVoiceMail, + ExportFileTopicsStream, ] From 8752bcff786f47a3e4663e3d505a9b06b7b36680 Mon Sep 17 00:00:00 2001 From: ynaim94-harrys Date: Thu, 22 Sep 2022 15:52:06 -0400 Subject: [PATCH 2/6] Use correct dev dependencies --- .github/workflows/ci_workflow.yml | 2 +- poetry.lock | 68 +++++++++++++++++++++++++++---- pyproject.toml | 7 ++-- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_workflow.yml b/.github/workflows/ci_workflow.yml index 7d9415e..6802b3d 100644 --- a/.github/workflows/ci_workflow.yml +++ b/.github/workflows/ci_workflow.yml @@ -23,7 +23,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1 with: - version: 1.2.1 + version: 1.2.0a2 - name: Install dependencies run: | poetry install diff --git a/poetry.lock b/poetry.lock index 0509893..57d75cb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,7 +32,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "black" version = "22.8.0" description = "The uncompromising code formatter." -category = "main" +category = "dev" optional = false python-versions = ">=3.6.2" @@ -42,6 +42,7 @@ mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} +typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] @@ -98,6 +99,7 @@ python-versions = ">=3.7" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" @@ -163,6 +165,7 @@ optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.7.0,<2.8.0" pyflakes = ">=2.3.0,<2.4.0" @@ -195,6 +198,7 @@ optional = false python-versions = ">=3.7" [package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] @@ -263,6 +267,7 @@ python-versions = "*" [package.dependencies] attrs = ">=17.4.0" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} pyrsistent = ">=0.14.0" setuptools = "*" six = ">=1.11.0" @@ -298,6 +303,7 @@ python-versions = ">=3.5" [package.dependencies] mypy-extensions = ">=0.4.3,<0.5.0" toml = "*" +typed-ast = {version = ">=1.4.0,<1.5.0", markers = "python_version < \"3.8\""} typing-extensions = ">=3.7.4" [package.extras] @@ -308,7 +314,7 @@ python2 = ["typed-ast (>=1.4.0,<1.5.0)"] name = "mypy-extensions" version = "0.4.3" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "main" +category = "dev" optional = false python-versions = "*" @@ -327,7 +333,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" name = "pathspec" version = "0.10.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "main" +category = "dev" optional = false python-versions = ">=3.7" @@ -366,7 +372,7 @@ dev = ["ipdb", "ipython", "nose", "pylint"] name = "platformdirs" version = "2.5.2" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" +category = "dev" optional = false python-versions = ">=3.7" @@ -382,6 +388,9 @@ category = "dev" optional = false python-versions = ">=3.6" +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] @@ -485,6 +494,7 @@ python-versions = ">=3.6" atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" @@ -583,6 +593,7 @@ python-versions = ">=3.7.1,<3.11" backoff = ">=1.8.0,<2.0" click = ">=8.0,<9.0" cryptography = ">=3.4.6,<38.0.0" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} inflection = ">=0.5.1,<0.6.0" joblib = ">=1.0.1,<2.0.0" jsonpath-ng = ">=1.5.3,<2.0.0" @@ -624,6 +635,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" [package.dependencies] greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -658,7 +670,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" name = "tomli" version = "1.2.3" description = "A lil' TOML parser" -category = "main" +category = "dev" optional = false python-versions = ">=3.6" @@ -673,6 +685,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] colorama = {version = ">=0.4.1", markers = "platform_system == \"Windows\""} filelock = ">=3.0.0" +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} packaging = ">=14" pluggy = ">=0.12.0" py = ">=1.4.17" @@ -684,6 +697,14 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2, docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] +[[package]] +name = "typed-ast" +version = "1.4.3" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-requests" version = "2.28.10" @@ -735,6 +756,7 @@ python-versions = ">=3.6" [package.dependencies] distlib = ">=0.3.5,<1" filelock = ">=3.4.1,<4" +importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.8\""} platformdirs = ">=2.4,<3" [package.extras] @@ -755,8 +777,8 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" -python-versions = "<3.11,>=3.8.1" -content-hash = "27ac360fe6eaebc2641cbeb4b3b6cb3fb24431e993c8a8032adf75bd25d32ef3" +python-versions = "<3.11,>=3.7.1" +content-hash = "e90339587568256be969679e8d9c8f3a757353915188e0f3012572c74af5b483" [metadata.files] atomicwrites = [ @@ -1249,6 +1271,38 @@ tox = [ {file = "tox-3.25.1-py2.py3-none-any.whl", hash = "sha256:c38e15f4733683a9cc0129fba078633e07eb0961f550a010ada879e95fb32632"}, {file = "tox-3.25.1.tar.gz", hash = "sha256:c138327815f53bc6da4fe56baec5f25f00622ae69ef3fe4e1e385720e22486f9"}, ] +typed-ast = [ + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, + {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, + {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, + {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, + {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, + {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, + {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, + {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, + {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, + {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, + {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, + {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, + {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, + {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, + {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, + {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, +] types-requests = [ {file = "types-requests-2.28.10.tar.gz", hash = "sha256:97d8f40aa1ffe1e58c3726c77d63c182daea9a72d9f1fa2cafdea756b2a19f2c"}, {file = "types_requests-2.28.10-py3-none-any.whl", hash = "sha256:45b485725ed58752f2b23461252f1c1ad9205b884a1e35f786bb295525a3e16a"}, diff --git a/pyproject.toml b/pyproject.toml index d9359c0..8659970 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,18 +13,19 @@ license = "Apache 2.0" python = "<3.11,>=3.7.1" requests = "^2.25.1" singer-sdk = "^0.10.0" -black = "^22.8.0" -importlib-metadata = "^4.12.0" [tool.poetry.dev-dependencies] pytest = "^6.2.5" tox = "^3.24.4" flake8 = "^3.9.2" -black = "^22.3.0" pydocstyle = "^6.1.1" mypy = "^0.910" types-requests = "^2.26.1" isort = "^5.10.1" +importlib-metadata = "^4.12.0" +black = "^22.8.0" + + [tool.isort] profile = "black" From 0ff738715c11bb1f68f99145ca5ec154857787fe Mon Sep 17 00:00:00 2001 From: ynaim94-harrys Date: Thu, 22 Sep 2022 16:03:12 -0400 Subject: [PATCH 3/6] restore poetry 1.2.1 in CI --- .github/workflows/ci_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_workflow.yml b/.github/workflows/ci_workflow.yml index 6802b3d..7d9415e 100644 --- a/.github/workflows/ci_workflow.yml +++ b/.github/workflows/ci_workflow.yml @@ -23,7 +23,7 @@ jobs: - name: Install Poetry uses: snok/install-poetry@v1 with: - version: 1.2.0a2 + version: 1.2.1 - name: Install dependencies run: | poetry install From 0b84795993baff3d861ba97cd52061c1c8546845 Mon Sep 17 00:00:00 2001 From: ynaim94-harrys Date: Thu, 22 Sep 2022 18:38:51 -0400 Subject: [PATCH 4/6] Use pendulum to handle dates --- poetry.lock | 2 +- pyproject.toml | 1 + tap_gladly/client.py | 2 -- tap_gladly/streams.py | 10 +++++----- tap_gladly/tests/test_core.py | 7 ++----- tap_gladly/tests/test_streams.py | 28 +++++++--------------------- 6 files changed, 16 insertions(+), 34 deletions(-) diff --git a/poetry.lock b/poetry.lock index 57d75cb..547000a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -778,7 +778,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "<3.11,>=3.7.1" -content-hash = "e90339587568256be969679e8d9c8f3a757353915188e0f3012572c74af5b483" +content-hash = "34e95066cefe5f6f0f1158a237ff02b05346a07f731bcb3580dd70e5e9f91749" [metadata.files] atomicwrites = [ diff --git a/pyproject.toml b/pyproject.toml index 8659970..5c0b438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ license = "Apache 2.0" python = "<3.11,>=3.7.1" requests = "^2.25.1" singer-sdk = "^0.10.0" +pendulum = "^2.1.2" [tool.poetry.dev-dependencies] pytest = "^6.2.5" diff --git a/tap_gladly/client.py b/tap_gladly/client.py index d6447de..4d984c7 100644 --- a/tap_gladly/client.py +++ b/tap_gladly/client.py @@ -13,8 +13,6 @@ class gladlyStream(RESTStream): """gladly stream class.""" - _common_date_format = "%Y-%m-%dT%H:%M:%SZ" - # OR use a dynamic url_base: @property def url_base(self) -> str: diff --git a/tap_gladly/streams.py b/tap_gladly/streams.py index ca468ab..f1e41b5 100644 --- a/tap_gladly/streams.py +++ b/tap_gladly/streams.py @@ -1,17 +1,16 @@ """Stream type classes for tap-gladly.""" import abc import json -from datetime import datetime from pathlib import Path from typing import Iterable, Optional +import pendulum import requests from singer_sdk import exceptions from singer_sdk.helpers.jsonpath import extract_jsonpath from tap_gladly.client import gladlyStream -# TODO: Delete this is if not using json files for schema definition SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") @@ -30,9 +29,10 @@ def post_process(self, row, context): """As needed, append or transform raw data to match expected structure.""" if "start_date" not in self.config: return row - if datetime.strptime( - row["parameters"]["startAt"], self._common_date_format - ) > datetime.strptime(self.config["start_date"], self._common_date_format): + + if pendulum.parse(row["parameters"]["startAt"]) > pendulum.parse( + self.config["start_date"] + ): return row return diff --git a/tap_gladly/tests/test_core.py b/tap_gladly/tests/test_core.py index 9942a82..b9dc3d5 100644 --- a/tap_gladly/tests/test_core.py +++ b/tap_gladly/tests/test_core.py @@ -1,16 +1,13 @@ """Tests standard tap features using the built-in SDK tests library.""" -import datetime +import pendulum from singer_sdk.testing import get_standard_tap_tests -from tap_gladly.client import gladlyStream from tap_gladly.tap import Tapgladly SAMPLE_CONFIG = { - "start_date": datetime.datetime.now(datetime.timezone.utc).strftime( - gladlyStream._common_date_format - ), + "start_date": pendulum.now().isoformat(), "username": "test", "password": "test", "api_url_base": "api_base_url", diff --git a/tap_gladly/tests/test_streams.py b/tap_gladly/tests/test_streams.py index d02df28..c8c04ee 100644 --- a/tap_gladly/tests/test_streams.py +++ b/tap_gladly/tests/test_streams.py @@ -2,14 +2,13 @@ import datetime -from tap_gladly.client import gladlyStream +import pendulum + from tap_gladly.streams import ExportFileConversationItemsChatMessage, ExportJobsStream from tap_gladly.tap import Tapgladly SAMPLE_CONFIG = { - "start_date": datetime.datetime.now(datetime.timezone.utc).strftime( - gladlyStream._common_date_format - ), + "start_date": pendulum.now(), "username": "test", "password": "test", "api_url_base": "api_base_url", @@ -20,10 +19,7 @@ def test_started_at(): tap_gladly = Tapgladly( config=dict( SAMPLE_CONFIG, - start_date=( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=3) - ).strftime(gladlyStream._common_date_format), + start_date=(pendulum.now() - datetime.timedelta(days=2)).isoformat(), ), parse_env_config=False, ) @@ -31,19 +27,12 @@ def test_started_at(): before_row = { "record": "data", "parameters": { - "startAt": ( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=3) - ).strftime(gladlyStream._common_date_format) + "startAt": (pendulum.now() - datetime.timedelta(days=3)).isoformat() }, } after_row = { "record": "data", - "parameters": { - "startAt": datetime.datetime.now(datetime.timezone.utc).strftime( - gladlyStream._common_date_format - ) - }, + "parameters": {"startAt": pendulum.now().isoformat()}, } assert not export_jobs_stream.post_process(before_row, None) assert export_jobs_stream.post_process(after_row, None) @@ -53,10 +42,7 @@ def test_filter_by_content_type(): tap_gladly = Tapgladly( config=dict( SAMPLE_CONFIG, - start_date=( - datetime.datetime.now(datetime.timezone.utc) - - datetime.timedelta(days=2) - ).strftime(gladlyStream._common_date_format), + start_date=(pendulum.now() - datetime.timedelta(days=2)).isoformat(), ), parse_env_config=False, ) From 7701b3787ef931efaf7e9ae9ed4a59dbacc8e307 Mon Sep 17 00:00:00 2001 From: ynaim94-harrys Date: Thu, 22 Sep 2022 22:13:27 -0400 Subject: [PATCH 5/6] Include start_date in the data we pull --- tap_gladly/client.py | 5 ----- tap_gladly/streams.py | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tap_gladly/client.py b/tap_gladly/client.py index 4d984c7..3d4f4f2 100644 --- a/tap_gladly/client.py +++ b/tap_gladly/client.py @@ -73,8 +73,3 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: """Parse the response and return an iterator of result records.""" # TODO: Parse response body and return a set of records. yield from extract_jsonpath(self.records_jsonpath, input=response.json()) - - def post_process(self, row, context): - """As needed, append or transform raw data to match expected structure.""" - # TODO: Delete this method if not needed. - return row diff --git a/tap_gladly/streams.py b/tap_gladly/streams.py index f1e41b5..228f74a 100644 --- a/tap_gladly/streams.py +++ b/tap_gladly/streams.py @@ -24,13 +24,12 @@ class ExportJobsStream(gladlyStream): # Optionally, you may also use `schema_filepath` in place of `schema`: schema_filepath = SCHEMAS_DIR / "export_jobs.json" - # start_date def post_process(self, row, context): """As needed, append or transform raw data to match expected structure.""" if "start_date" not in self.config: return row - if pendulum.parse(row["parameters"]["startAt"]) > pendulum.parse( + if pendulum.parse(row["parameters"]["startAt"]) >= pendulum.parse( self.config["start_date"] ): return row From 463d7df0cabb4d7dcc6ec5f8bda981301210d850 Mon Sep 17 00:00:00 2001 From: ynaim94-harrys Date: Thu, 22 Sep 2022 22:45:44 -0400 Subject: [PATCH 6/6] Only extract from completed jobs --- tap_gladly/streams.py | 15 ++++++--------- tap_gladly/tap.py | 4 ++-- tap_gladly/tests/test_streams.py | 11 +++++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tap_gladly/streams.py b/tap_gladly/streams.py index 228f74a..98fed34 100644 --- a/tap_gladly/streams.py +++ b/tap_gladly/streams.py @@ -14,22 +14,19 @@ SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") -class ExportJobsStream(gladlyStream): +class ExportCompletedJobsStream(gladlyStream): """List export jobs stream.""" name = "jobs" - path = "/export/jobs" + path = "/export/jobs?status=COMPLETED" primary_keys = ["id"] replication_key = None # Optionally, you may also use `schema_filepath` in place of `schema`: schema_filepath = SCHEMAS_DIR / "export_jobs.json" def post_process(self, row, context): - """As needed, append or transform raw data to match expected structure.""" - if "start_date" not in self.config: - return row - - if pendulum.parse(row["parameters"]["startAt"]) >= pendulum.parse( + """Filter jobs that finished before start_date.""" + if pendulum.parse(row["parameters"]["endAt"]) >= pendulum.parse( self.config["start_date"] ): return row @@ -47,7 +44,7 @@ class ExportFileTopicsStream(gladlyStream): path = "/export/jobs/{job_id}/files/topics.jsonl" primary_keys = ["id"] replication_key = None - parent_stream_type = ExportJobsStream + parent_stream_type = ExportCompletedJobsStream ignore_parent_replication_key = True schema_filepath = SCHEMAS_DIR / "export_topics.json" @@ -64,7 +61,7 @@ class ExportFileConversationItemsStream(gladlyStream, abc.ABC): path = "/export/jobs/{job_id}/files/conversation_items.jsonl" primary_keys = ["id"] replication_key = None - parent_stream_type = ExportJobsStream + parent_stream_type = ExportCompletedJobsStream ignore_parent_replication_key = True @property diff --git a/tap_gladly/tap.py b/tap_gladly/tap.py index 82aedf5..07bfe25 100644 --- a/tap_gladly/tap.py +++ b/tap_gladly/tap.py @@ -7,6 +7,7 @@ # TODO: Import your custom stream types here: from tap_gladly.streams import ( + ExportCompletedJobsStream, ExportFileConversationItemsChatMessage, ExportFileConversationItemsConversationNote, ExportFileConversationItemsConversationStatusChange, @@ -15,11 +16,10 @@ ExportFileConversationItemsTopicChange, ExportFileConversationItemsVoiceMail, ExportFileTopicsStream, - ExportJobsStream, ) STREAM_TYPES = [ - ExportJobsStream, + ExportCompletedJobsStream, ExportFileConversationItemsChatMessage, ExportFileConversationItemsConversationNote, ExportFileConversationItemsTopicChange, diff --git a/tap_gladly/tests/test_streams.py b/tap_gladly/tests/test_streams.py index c8c04ee..801fa21 100644 --- a/tap_gladly/tests/test_streams.py +++ b/tap_gladly/tests/test_streams.py @@ -4,7 +4,10 @@ import pendulum -from tap_gladly.streams import ExportFileConversationItemsChatMessage, ExportJobsStream +from tap_gladly.streams import ( + ExportCompletedJobsStream, + ExportFileConversationItemsChatMessage, +) from tap_gladly.tap import Tapgladly SAMPLE_CONFIG = { @@ -23,16 +26,16 @@ def test_started_at(): ), parse_env_config=False, ) - export_jobs_stream = ExportJobsStream(tap_gladly) + export_jobs_stream = ExportCompletedJobsStream(tap_gladly) before_row = { "record": "data", "parameters": { - "startAt": (pendulum.now() - datetime.timedelta(days=3)).isoformat() + "endAt": (pendulum.now() - datetime.timedelta(days=3)).isoformat() }, } after_row = { "record": "data", - "parameters": {"startAt": pendulum.now().isoformat()}, + "parameters": {"endAt": pendulum.now().isoformat()}, } assert not export_jobs_stream.post_process(before_row, None) assert export_jobs_stream.post_process(after_row, None)