do not merge - additional resources support

.gitignore

@@ -1,6 +1,7 @@
 datadog_sync/version.py
 **/*.pyc
 **/*.pyo
+**/*~
 .vscode/
 .idea/
 .coverage

README.md

@@ -1,3 +1,79 @@
+# Deepomatic fork
+Dump of additional hackish import/sync/cleanup_org for extra resources not supported by upstream, used by Deepomatic when migrating region. Maybe it will help somebody.
+
+Inspiration: https://careers.wolt.com/en/blog/tech/datadog-migration-wolt
+
+For some resources: it uses non-official api (from web frontend), using `dogweb` cookie and `x-csrf-token` header
+```
+source_cookie_dogweb="xxx"
+destination_cookie_dogweb="xxx"
+destination_csrf_token="xxx"
+```
+Warning: it's a hack, with shortcuts:
+- it is *not* endorsed by Datadog (or supported by Deepomatic)
+- authentication is either/or: cookie_dogweb config are required for those resources, and datadog-cli switches to cookie dogweb mode if config set, it *will not* work for other resources
+- web frontend api is not documented, it could break at any time
+
+
+## extra resources
+### logs_facets
+how to use:
+- edit hardcoded `sourceid` in `datadog_sync/model/logs_facets.py` for your organizations, by getting the values in URLs with manual update facet on the web ui.
+- setup dogweb cookie mode, cf above
+
+### logs_views
+how to use:
+- setup dogweb cookie mode, cf above
+
+### metric_metadatas
+create metric metadata is *not* supported by datadog api, we can just update it on already existing metric.
+- first push data-points on metric, then rerun the script when new metrics are populated
+
+### incidents
+The supported scenario is importing all incidents (in order) so `public_id` (1, 2, etc.) are identical in source & destination organizations: never create new incidents in the destination organization before finishing the migration with datadog-sync-cli.
+
+Only the base incident data is supported, related resources (integrations(slack), todos(remediations), attachments) may be done later with dedicated resources.
+
+The import is lossy: for example the creation date is on sync, timeline is lost, etc.
+
+'notifications' explicitly not-sync'ed to avoid spamming people during import (although later tests seem to conclude 'inactive' user (invitation pending: sync'ed users, but they never connected to the destination region) are *not* notified)
+
+### incidents_integrations
+- api bug: it url-escapes slack `redirect_url` `&` query-string separator character before saving: this leads to a forever diff: datadog-sync-cli tries to PATCH the correct value on each sync, the server saves a wrong value.
+
+### incidents_todos
+- creation date & author is lost, as usual
+
+### incident_org_settings
+- undocumented api, but standard v2 api used by web frontend, works with API/APP key
+- just one resource per org, forcing update, ignoring ids, etc.
+
+### incidents_config_fields
+- perpetual diff: on 'metadata' for ootb service & team:
+  - PATCH ok (maybe ignores metadata?)
+  - but PATCH response contains `metadata: null`
+  => `diffs` always shows it; it's ok, we can ignore those
+
+### incidents_config_notifications_templates
+
+### incidents_config_integrations_workflows
+Covers General>Integrations & Notifications>Rules
+- (api inconsistency: `attributes.triggers.variables.severity_values` and `attributes.triggers.variables.status_values` are `null` in read calls, and require an array in write calls)
+- errors (probably because some workflows are hardcoded, not duplicable, but no obvious attribute to distingish them)
+  - Error: 400 Bad Request - {"errors":["a workflow like that already exists"]}
+  - Error: 400 Bad Request - {"errors":["Invalid payload: 'name' is invalid"]}
+  => ignoring those errors for now, and manually fixed `Send all incident updates to a global channel` via web frontend.
+
+### integrations_slack_channels
+how to use:
+- supports only *one* slack account
+- api doesn't support `muting` option
+- manually create the slack integration in destination organization, with *same name* as in source
+- edit hardcoded `slack_account_name` in `datadog_sync/model/integrations_slack_channels.py` for your organizations
+- run import & diffs & sync as usual
+
+---
+
 # datadog-sync-cli
 Datadog cli tool to sync resources across organizations.
 

datadog_sync/commands/shared/options.py

@@ -51,6 +51,13 @@ def handle_parse_result(self, ctx: Context, opts: Dict[Any, Any], args: List[Any
         help="Datadog source organization API url.",
         cls=CustomOptionClass,
     ),
+    option(
+        "--source-cookie-dogweb",
+        envvar=constants.DD_SOURCE_COOKIE_DOGWEB,
+        required=False,
+        help="Datadog source organization 'dogweb' cookie.",
+        cls=CustomOptionClass,
+    ),
 ]
 
 _destination_auth_options = [
@@ -77,6 +84,20 @@ def handle_parse_result(self, ctx: Context, opts: Dict[Any, Any], args: List[Any
         help="Datadog destination organization API url.",
         cls=CustomOptionClass,
     ),
+    option(
+        "--destination-cookie-dogweb",
+        envvar=constants.DD_DESTINATION_COOKIE_DOGWEB,
+        required=False,
+        help="Datadog destination organization 'dogweb' cookie.",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--destination-csrf-token",
+        envvar=constants.DD_DESTINATION_CSRF_TOKEN,
+        required=False,
+        help="Datadog destination organization 'x-csrf-token' header.",
+        cls=CustomOptionClass,
+    ),
 ]
 
 

datadog_sync/constants.py

@@ -7,9 +7,12 @@
 DD_SOURCE_API_URL = "DD_SOURCE_API_URL"
 DD_SOURCE_API_KEY = "DD_SOURCE_API_KEY"
 DD_SOURCE_APP_KEY = "DD_SOURCE_APP_KEY"
+DD_SOURCE_COOKIE_DOGWEB = "DD_SOURCE_COOKIE_DOGWEB"
 DD_DESTINATION_API_URL = "DD_DESTINATION_API_URL"
 DD_DESTINATION_API_KEY = "DD_DESTINATION_API_KEY"
 DD_DESTINATION_APP_KEY = "DD_DESTINATION_APP_KEY"
+DD_DESTINATION_COOKIE_DOGWEB = "DD_DESTINATION_COOKIE_DOGWEB"
+DD_DESTINATION_CSRF_TOKEN = "DD_DESTINATION_CSRF_TOKEN"
 DD_HTTP_CLIENT_RETRY_TIMEOUT = "DD_HTTP_CLIENT_RETRY_TIMEOUT"
 DD_HTTP_CLIENT_TIMEOUT = "DD_HTTP_CLIENT_TIMEOUT"
 DD_RESOURCES = "DD_RESOURCES"
@@ -30,6 +33,7 @@
 SOURCE_ORIGIN = "source"
 DESTINATION_ORIGIN = "destination"
 VALIDATE_ENDPOINT = "/api/v1/validate"
+VALIDATE_ENDPOINT_COOKIEAUTH = "/api/v1/settings/favorite/list"
 
 # Commands
 CMD_IMPORT = "import"

datadog_sync/model/incident_org_settings.py

@@ -0,0 +1,63 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the 3-clause BSD style license (see LICENSE).
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, Optional, List, Dict, cast
+
+from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
+
+if TYPE_CHECKING:
+    from datadog_sync.utils.custom_client import CustomClient
+
+
+class IncidentOrgSettings(BaseResource):
+    resource_type = "incident_org_settings"
+    resource_config = ResourceConfig(
+        base_path="/api/v2/incidents/config/org/settings",
+        excluded_attributes=[
+            "id",
+            "attributes.modified",
+        ]
+    )
+    # Additional Incidents specific attributes
+
+    def get_resources(self, client: CustomClient) -> List[Dict]:
+        resp = client.get(self.resource_config.base_path).json()["data"]
+        return [ resp ]
+
+    def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> None:
+        if _id:
+            # there is only one settings, ignoring id
+            source_client = self.config.source_client
+            resource = source_client.get(self.resource_config.base_path).json()["data"]
+
+        resource = cast(dict, resource)
+        self.resource_config.source_resources[resource["id"]] = resource
+
+    def pre_resource_action_hook(self, _id, resource: Dict) -> None:
+        pass
+
+    def pre_apply_hook(self) -> None:
+        pass
+
+    def create_resource(self, _id: str, resource: Dict) -> None:
+        # the settings is always there, just update
+        self.update_resource(_id, resource)
+
+    def update_resource(self, _id: str, resource: Dict) -> None:
+        destination_client = self.config.destination_client
+        payload = {"data": resource}
+        resp = destination_client.patch(
+            self.resource_config.base_path,
+            payload,
+        ).json()["data"]
+
+        self.resource_config.destination_resources[_id] = resp
+
+    def delete_resource(self, _id: str) -> None:
+        raise Exception("deleting incident_org_settings is not supported")
+
+    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
+        pass

datadog_sync/model/incidents.py

@@ -0,0 +1,130 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the 3-clause BSD style license (see LICENSE).
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, Optional, List, Dict, cast
+
+from datadog_sync.utils.base_resource import BaseResource, ResourceConfig
+from datadog_sync.utils.custom_client import PaginationConfig
+
+if TYPE_CHECKING:
+    from datadog_sync.utils.custom_client import CustomClient
+
+
+class Incidents(BaseResource):
+    resource_type = "incidents"
+    resource_config = ResourceConfig(
+        resource_connections={
+            "users": [
+                "relationships.commander_user.data.id",
+            ]
+        },
+        base_path="/api/v2/incidents",
+        excluded_attributes=[
+            "id",
+            "attributes.public_id",
+            "attributes.commander",  # somehow returned by create or update, not by get
+            "attributes.last_modified_by",  # somehow returned by create or update, not by get
+            "attributes.last_modified_by_uuid",
+            "attributes.created",
+            "attributes.modified",
+            "attributes.created_by",  # somehow returned by create or update, not by get
+            "attributes.created_by_uuid",
+            "attributes.notification_handles",  # too hard to support properly, also, it gives wrong dates, and possibly spams people, we don't want that; ok to loose that info
+            "attributes.time_to_resolve",
+            "attributes.customer_impact_duration",  # computed field
+            "relationships.created_by_user",
+            "relationships.last_modified_by_user",
+            "relationships.user_defined_fields",
+            "relationships.integrations",
+            "relationships.attachments",
+            "relationships.responders",
+            "relationships.impacts",
+        ],
+        non_nullable_attr=[
+            "attributes.creation_idempotency_key",
+            "attributes.customer_impact_scope",
+        ],
+
+    )
+    # Additional Incidents specific attributes
+    pagination_config = PaginationConfig(
+        page_size=100,
+        page_number_param="page[offset]",
+        page_size_param="page[size]",
+        # this endpoint uses offset (number of items) instead of page number, workaround the paginated client by reusing `page_number` to store offset instead (computed here because we don't have `resp`)
+        page_number_func=lambda idx, page_size, page_number: page_size * (idx + 1),
+        # just return 1, the pagination loop already handles breaking when a page is smaller than page size
+        remaining_func=lambda *args: 1,
+    )
+
+    def get_resources(self, client: CustomClient) -> List[Dict]:
+        # we return the incidents in public_id order, so creating them on a fresh organizations will gives us the same public_id in source & destination organizations
+
+        resp = client.paginated_request(client.get)(
+            self.resource_config.base_path, pagination_config=self.pagination_config
+        )
+
+        return resp
+
+    def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> None:
+        if _id:
+            source_client = self.config.source_client
+            resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"]
+
+        resource = cast(dict, resource)
+
+        # it's the new default imposed by the api; forcing it here so we don't have a forever-diff
+        if "visibility" in resource["attributes"] and resource["attributes"]["visibility"] is None:
+            resource["attributes"]["visibility"] = "organization"
+
+        # let's do some deepomatic-specific incidents fields migrations:
+        if "Namespace" in resource["attributes"]["fields"] and resource["attributes"]["fields"]["Namespace"]["value"] is not None and "kube_namespace" in resource["attributes"]["fields"] and resource["attributes"]["fields"]["kube_namespace"]["value"] is None:
+            resource["attributes"]["fields"]["kube_namespace"]["value"] = resource["attributes"]["fields"]["Namespace"]["value"]
+            resource["attributes"]["fields"]["Namespace"]["value"] = None
+
+        self.resource_config.source_resources[resource["id"]] = resource
+
+    def pre_resource_action_hook(self, _id, resource: Dict) -> None:
+        pass
+
+    def pre_apply_hook(self) -> None:
+        pass
+
+    def create_resource(self, _id: str, resource: Dict) -> None:
+        destination_client = self.config.destination_client
+        payload = {"data": resource}
+        # the datadog api documentation says only a subset of accepted fields for creation; in practice it does handles only a subset, and ignores the others
+        resp = destination_client.post(
+            self.resource_config.base_path,
+            payload,
+        ).json()
+
+        self.resource_config.destination_resources[_id] = resp["data"]
+
+        # create doesn't accept everything right away, e.g. attributes.resolved; follow the create by an update to sync more data
+        self.update_resource(_id, resource)
+
+    def update_resource(self, _id: str, resource: Dict) -> None:
+        destination_client = self.config.destination_client
+        payload = {"data": resource}
+
+        resp = destination_client.patch(
+            self.resource_config.base_path +
+            f"/{self.resource_config.destination_resources[_id]['id']}",
+            payload,
+        ).json()
+
+        self.resource_config.destination_resources[_id] = resp["data"]
+
+    def delete_resource(self, _id: str) -> None:
+        destination_client = self.config.destination_client
+        destination_client.delete(
+            self.resource_config.base_path +
+            f"/{self.resource_config.destination_resources[_id]['id']}"
+        )
+
+    def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
+        return super(Incidents, self).connect_id(key, r_obj, resource_to_connect)