diff --git a/hexa/catalog/views.py b/hexa/catalog/views.py index 0270bc35e..ef954d342 100644 --- a/hexa/catalog/views.py +++ b/hexa/catalog/views.py @@ -60,6 +60,7 @@ def search(request: HttpRequest) -> HttpResponse: ) +# TODO: post-only? def datasource_sync( request: HttpRequest, datasource_contenttype_id: int, datasource_id: uuid.UUID ): diff --git a/hexa/plugins/connector_dhis2/templates/connector_dhis2/data_element_list.html b/hexa/plugins/connector_dhis2/templates/connector_dhis2/data_element_list.html index b1b7e5da8..38fbb84c3 100644 --- a/hexa/plugins/connector_dhis2/templates/connector_dhis2/data_element_list.html +++ b/hexa/plugins/connector_dhis2/templates/connector_dhis2/data_element_list.html @@ -6,7 +6,7 @@ {% block page_content %} {% embed "ui/section/section.html" %} {% slot header %} - {% include "ui/section/section_heading_with_label.html" with title=section_title label=section_label %} + {% include "ui/section/heading.html" with title=section_title label=section_label %} {% endslot %} {% slot content %} {{ data_element_grid }} diff --git a/hexa/plugins/connector_dhis2/templates/connector_dhis2/dataset_list.html b/hexa/plugins/connector_dhis2/templates/connector_dhis2/dataset_list.html index 84880aaa8..c68005bc5 100644 --- a/hexa/plugins/connector_dhis2/templates/connector_dhis2/dataset_list.html +++ b/hexa/plugins/connector_dhis2/templates/connector_dhis2/dataset_list.html @@ -6,7 +6,7 @@ {% block page_content %} {% embed "ui/section/section.html" %} {% slot header %} - {% include "ui/section/section_heading_with_label.html" with title=section_title label=section_label %} + {% include "ui/section/heading.html" with title=section_title label=section_label %} {% endslot %} {% slot content %} {{ dataset_grid }} diff --git a/hexa/plugins/connector_dhis2/templates/connector_dhis2/indicator_list.html b/hexa/plugins/connector_dhis2/templates/connector_dhis2/indicator_list.html index 2f411460b..c48037753 100644 --- a/hexa/plugins/connector_dhis2/templates/connector_dhis2/indicator_list.html +++ b/hexa/plugins/connector_dhis2/templates/connector_dhis2/indicator_list.html @@ -6,7 +6,7 @@ {% block page_content %} {% embed "ui/section/section.html" %} {% slot header %} - {% include "ui/section/section_heading_with_label.html" with title=section_title label=section_label %} + {% include "ui/section/heading.html" with title=section_title label=section_label %} {% endslot %} {% slot content %} {{ indicator_grid }} diff --git a/hexa/plugins/connector_postgresql/templates/connector_postgresql/table_list.html b/hexa/plugins/connector_postgresql/templates/connector_postgresql/table_list.html index c72e02919..8fdbdc709 100644 --- a/hexa/plugins/connector_postgresql/templates/connector_postgresql/table_list.html +++ b/hexa/plugins/connector_postgresql/templates/connector_postgresql/table_list.html @@ -5,7 +5,7 @@ {% block page_content %} {% embed "ui/section/section.html" %} {% slot header %} - {% include "ui/section/section_heading_with_label.html" with title=section_title label=section_label %} + {% include "ui/section/heading.html" with title=section_title label=section_label %} {% endslot %} {% slot content %} {{ table_grid }} diff --git a/hexa/plugins/connector_s3/api.py b/hexa/plugins/connector_s3/api.py index 4c0352fa7..f7050a642 100644 --- a/hexa/plugins/connector_s3/api.py +++ b/hexa/plugins/connector_s3/api.py @@ -5,6 +5,7 @@ import boto3 import stringcase +from botocore.config import Config import hexa.plugins.connector_s3.models import hexa.user_management.models @@ -70,3 +71,71 @@ def generate_sts_buckets_credentials( ) return response["Credentials"] + + +def _build_s3_client( + *, + principal_credentials: hexa.plugins.connector_s3.models.Credentials, + bucket: hexa.plugins.connector_s3.models.Bucket, + user: hexa.user_management.models.User | None = None, +): + sts_credentials = generate_sts_buckets_credentials( + user=user, + principal_credentials=principal_credentials, + buckets=[bucket], + duration=900, + ) + return boto3.client( + "s3", + principal_credentials.default_region, + aws_access_key_id=sts_credentials["AccessKeyId"], + aws_secret_access_key=sts_credentials["SecretAccessKey"], + aws_session_token=sts_credentials["SessionToken"], + config=Config(signature_version="s3v4"), + ) + + +def head_bucket( + *, + principal_credentials: hexa.plugins.connector_s3.models.Credentials, + bucket: hexa.plugins.connector_s3.models.Bucket, +): + s3_client = _build_s3_client( + principal_credentials=principal_credentials, bucket=bucket + ) + + return s3_client.head_bucket(bucket.name) + + +def generate_download_url( + *, + principal_credentials: hexa.plugins.connector_s3.models.Credentials, + bucket: hexa.plugins.connector_s3.models.Bucket, + target_object: hexa.plugins.connector_s3.models.Object, +): + s3_client = _build_s3_client( + principal_credentials=principal_credentials, bucket=bucket + ) + + return s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": bucket.name, "Key": target_object.key}, + ExpiresIn=60 * 10, + ) + + +def generate_upload_url( + *, + principal_credentials: hexa.plugins.connector_s3.models.Credentials, + bucket: hexa.plugins.connector_s3.models.Bucket, + target_key: str, +): + s3_client = _build_s3_client( + principal_credentials=principal_credentials, bucket=bucket + ) + + return s3_client.generate_presigned_url( + "put_object", + Params={"Bucket": bucket.name, "Key": target_key}, + ExpiresIn=60 * 60, + ) diff --git a/hexa/plugins/connector_s3/models.py b/hexa/plugins/connector_s3/models.py index a10f466c7..ef42f91b2 100644 --- a/hexa/plugins/connector_s3/models.py +++ b/hexa/plugins/connector_s3/models.py @@ -24,7 +24,7 @@ from hexa.catalog.sync import DatasourceSyncResult from hexa.core.models import Permission from hexa.core.models.cryptography import EncryptedTextField -from hexa.plugins.connector_s3.api import generate_sts_buckets_credentials +from hexa.plugins.connector_s3.api import generate_sts_buckets_credentials, head_bucket class Credentials(Base): @@ -74,48 +74,27 @@ class Meta: objects = BucketQuerySet.as_manager() - def get_boto_client(self): + @property + def principal_credentials(self): try: - principal_s3_credentials = Credentials.objects.get() + return Credentials.objects.get() except (Credentials.DoesNotExist, Credentials.MultipleObjectsReturned): raise ValidationError( - "Ensure the S3 connector plugin first has a single credentials entry" + "The S3 connector plugin should be configured with a single Credentials entry" ) - sts_credentials = generate_sts_buckets_credentials( - user=None, - principal_credentials=principal_s3_credentials, - buckets=[self], - duration=900, - ) - return boto3.client( - "s3", - principal_s3_credentials.default_region, - aws_access_key_id=sts_credentials["AccessKeyId"], - aws_secret_access_key=sts_credentials["SecretAccessKey"], - aws_session_token=sts_credentials["SessionToken"], - config=Config(signature_version="s3v4"), - ) - def clean(self): try: - self.get_boto_client().head_bucket(Bucket=self.name) + head_bucket(self.principal_credentials, self) except ClientError as e: raise ValidationError(e) def sync(self): # TODO: move in api/sync module """Sync the bucket by querying the S3 API""" - try: - principal_s3_credentials = Credentials.objects.get() - except (Credentials.DoesNotExist, Credentials.MultipleObjectsReturned): - raise ValueError( - "Your s3 connector plugin should have a single credentials entry" - ) - sts_credentials = generate_sts_buckets_credentials( user=None, - principal_credentials=principal_s3_credentials, + principal_credentials=self.principal_credentials, buckets=[self], ) fs = S3FileSystem( diff --git a/hexa/plugins/connector_s3/templates/connector_s3/components/upload.html b/hexa/plugins/connector_s3/templates/connector_s3/components/upload.html new file mode 100644 index 000000000..e14cb77de --- /dev/null +++ b/hexa/plugins/connector_s3/templates/connector_s3/components/upload.html @@ -0,0 +1,17 @@ +{% load i18n %} + +
+
+ +
+
diff --git a/hexa/plugins/connector_s3/templates/connector_s3/datasource_detail.html b/hexa/plugins/connector_s3/templates/connector_s3/datasource_detail.html index 863174607..50c1c89f2 100644 --- a/hexa/plugins/connector_s3/templates/connector_s3/datasource_detail.html +++ b/hexa/plugins/connector_s3/templates/connector_s3/datasource_detail.html @@ -7,6 +7,15 @@ {{ bucket_card }} {% include "comments/components/page_section_comments.html" with object=datasource %} {% embed "ui/section/section.html" with title=_("Objects") %} + {% slot extra_wrapper_attrs %} + x-data="S3Upload('{% url "connector_s3:object_upload" datasource.id %}', '{{ sync_url }}')" + x-init="init($el)" + x-swap="objects_section" + x-html="refreshedHtml" + {% endslot %} + {% slot actions %} + {% include "connector_s3/components/upload.html" with bucket=datasource sync_url=sync_url %} + {% endslot %} {% slot content %} {{ datagrid }} {% endslot %} diff --git a/hexa/plugins/connector_s3/templates/connector_s3/object_detail.html b/hexa/plugins/connector_s3/templates/connector_s3/object_detail.html index e10fc9b6a..ca5e3e3f2 100644 --- a/hexa/plugins/connector_s3/templates/connector_s3/object_detail.html +++ b/hexa/plugins/connector_s3/templates/connector_s3/object_detail.html @@ -7,6 +7,15 @@ {% include "comments/components/page_section_comments.html" with object=object %} {% if object.type == 'directory' %} {% embed "ui/section/section.html" with title=_("Objects") %} + {% slot extra_wrapper_attrs %} + x-data="S3Upload('{% url "connector_s3:object_upload" datasource.id %}', '{{ sync_url }}', '{{ object.key }}')" + x-init="init($el)" + x-swap="objects_section" + x-html="refreshedHtml" + {% endslot %} + {% slot actions %} + {% include "connector_s3/components/upload.html" with bucket=datasource sync_url=sync_url %} + {% endslot %} {% slot content %} {{ datagrid }} {% endslot %} diff --git a/hexa/plugins/connector_s3/tests/test_api.py b/hexa/plugins/connector_s3/tests/test_api.py new file mode 100644 index 000000000..68b9051ae --- /dev/null +++ b/hexa/plugins/connector_s3/tests/test_api.py @@ -0,0 +1,52 @@ +import boto3 +from django import test +from moto import mock_s3, mock_sts + +from hexa.plugins.connector_s3.api import generate_download_url, generate_upload_url +from hexa.plugins.connector_s3.models import Bucket, Credentials, Object + + +class ApiTest(test.TestCase): + bucket_name = "test-bucket" + + def setUp(self): + self.credentials = Credentials.objects.create( + username="test-username", + role_arn="test-arn-arn-arn-arn", + default_region="eu-central-1", + ) + self.bucket = Bucket.objects.create(name=self.bucket_name) + + @mock_s3 + @mock_sts + def test_generate_download_url(self): + s3_client = boto3.client("s3") + s3_client.create_bucket(Bucket="test-bucket") + s3_client.put_object(Bucket="test-bucket", Key="test.csv", Body="test") + + target_object = Object.objects.create( + bucket=self.bucket, key="test.csv", size=100 + ) + + self.assertIsInstance( + generate_download_url( + principal_credentials=self.credentials, + bucket=self.bucket, + target_object=target_object, + ), + str, + ) + + @mock_s3 + @mock_sts + def test_generate_upload_url(self): + s3_client = boto3.client("s3") + s3_client.create_bucket(Bucket="test-bucket") + self.assertIsInstance( + generate_upload_url( + principal_credentials=self.credentials, + bucket=self.bucket, + target_key="test.csv", + ), + str, + ) diff --git a/hexa/plugins/connector_s3/urls.py b/hexa/plugins/connector_s3/urls.py index f542d50a7..7780219aa 100644 --- a/hexa/plugins/connector_s3/urls.py +++ b/hexa/plugins/connector_s3/urls.py @@ -16,4 +16,9 @@ views.object_download, name="object_download", ), + path( + "/object_upload/", + views.object_upload, + name="object_upload", + ), ] diff --git a/hexa/plugins/connector_s3/views.py b/hexa/plugins/connector_s3/views.py index f79d31071..ce0927271 100644 --- a/hexa/plugins/connector_s3/views.py +++ b/hexa/plugins/connector_s3/views.py @@ -2,8 +2,10 @@ from django.http import HttpRequest, HttpResponse from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from django.utils.translation import ugettext_lazy as _ +from .api import generate_download_url, generate_upload_url from .datacards import BucketCard, ObjectCard from .datagrids import ObjectGrid from .models import Bucket, Object @@ -28,11 +30,24 @@ def datasource_detail(request: HttpRequest, datasource_id: uuid.UUID) -> HttpRes page=int(request.GET.get("page", "1")), ) + # TODO: discuss + # Shouldn't we place that on datasource / entry models? Or at least a helper function + # alternative: sync by index_id? weird but practical + # alternative2: upload as template tag + sync_url = reverse( + "catalog:datasource_sync", + kwargs={ + "datasource_id": bucket.id, + "datasource_contenttype": ContentType.objects.get_for_model(Bucket).id, + }, + ) + return render( request, "connector_s3/datasource_detail.html", { "datasource": bucket, + "sync_url": sync_url, "breadcrumbs": breadcrumbs, "bucket_card": bucket_card, "datagrid": datagrid, @@ -72,12 +87,22 @@ def object_detail( page=int(request.GET.get("page", "1")), ) + # TODO: duplicated with above block + sync_url = reverse( + "catalog:datasource_sync", + kwargs={ + "datasource_id": bucket.id, + "datasource_contenttype": ContentType.objects.get_for_model(Bucket).id, + }, + ) + return render( request, "connector_s3/object_detail.html", { "datasource": bucket, "object": s3_object, + "sync_url": sync_url, "object_card": object_card, "breadcrumbs": breadcrumbs, "datagrid": datagrid, @@ -91,12 +116,25 @@ def object_download( bucket = get_object_or_404( Bucket.objects.filter_for_user(request.user), pk=bucket_id ) - s3_object = get_object_or_404(bucket.object_set.all(), key=path) + target_object = get_object_or_404(bucket.object_set.all(), key=path) - response = bucket.get_boto_client().generate_presigned_url( - "get_object", - Params={"Bucket": s3_object.bucket.name, "Key": s3_object.key}, - ExpiresIn=60 * 10, + download_url = generate_download_url( + principal_credentials=bucket.principal_credentials, + bucket=bucket, + target_object=target_object, + ) + + return redirect(download_url) + + +def object_upload(request, bucket_id): + bucket = get_object_or_404( + Bucket.objects.filter_for_user(request.user), pk=bucket_id + ) + upload_url = generate_upload_url( + principal_credentials=bucket.principal_credentials, + bucket=bucket, + target_key=request.GET["object_key"], ) - return redirect(response) + return HttpResponse(upload_url, status=201) diff --git a/hexa/static/js/hexa.js b/hexa/static/js/hexa.js index 6adb377f8..d16c1400b 100644 --- a/hexa/static/js/hexa.js +++ b/hexa/static/js/hexa.js @@ -298,4 +298,64 @@ function TomSelectable(multiple = true) { }) } } -} \ No newline at end of file +} + +// TODO: should be placed in s3 app +function S3Upload(getUploadUrl, syncUrl, prefix="") { + return { + refreshedHtml: null, + uploading: false, + init($el) { + this.refreshedHtml = $el.innerHTML; + }, + async onChange() { + this.uploading = true; + const uploadedFile = this.$refs.input.files[0]; + const fileName = uploadedFile.name; + + // Get presigned upload URL + const signedUrlResponse = await fetch(`${getUploadUrl}?object_key=${prefix}${fileName}`, { + method: this.$refs.form.method, + headers: { + 'Content-Type': 'text/plain', + "X-CSRFToken": document.cookie + .split('; ') + .find(row => row.startsWith('csrftoken=')) + .split('=')[1] + }, + }); + if (signedUrlResponse.status !== 201) { + console.error(`Error when generating presigned URL (code: ${signedUrlResponse.status})`); + return; + } + const preSignedUrl = await signedUrlResponse.text(); + + // Upload file + const uploadResponse = await fetch(preSignedUrl, { + method: "PUT", + headers: { + "Content-Type": uploadedFile.type, + }, + body: uploadedFile + }); + if (uploadResponse.status !== 200) { + console.error(`Error when uploading file to S3 (code: ${uploadResponse.status})`); + return; + } + + // Sync & refresh section + const refreshedResponse = await fetch(syncUrl, { + method: "GET", + }); + if (refreshedResponse.status !== 200) { + console.error(`Error when refreshing after sync (code: ${refreshedResponse.status})`); + return; + } + const frag = document.createElement("div"); + frag.innerHTML = await refreshedResponse.text(); + const swap = frag.querySelector("[x-swap=objects_section]"); + this.refreshedHtml = swap.innerHTML; + this.uploading = false; + } + } +} diff --git a/hexa/ui/templates/ui/icons/refresh.html b/hexa/ui/templates/ui/icons/refresh.html index c28f0a024..e8c74c08a 100644 --- a/hexa/ui/templates/ui/icons/refresh.html +++ b/hexa/ui/templates/ui/icons/refresh.html @@ -1,5 +1,5 @@ + + + diff --git a/hexa/ui/templates/ui/section/heading.html b/hexa/ui/templates/ui/section/heading.html new file mode 100644 index 000000000..65b49d451 --- /dev/null +++ b/hexa/ui/templates/ui/section/heading.html @@ -0,0 +1,14 @@ +{% load embed %} +
+
+

+ {{ title }} +

+ {% if label %} +

{{ label }}

+ {% endif %} +
+
+ {% slot actions %}{% endslot %} +
+
diff --git a/hexa/ui/templates/ui/section/section.html b/hexa/ui/templates/ui/section/section.html index 7c4de813a..8fbb866cd 100644 --- a/hexa/ui/templates/ui/section/section.html +++ b/hexa/ui/templates/ui/section/section.html @@ -1,12 +1,12 @@ {% load embed %}
{% slot header %} - {% include "ui/section/heading_simple.html" with title=title|default:_("Untitled section") %} + {% include "ui/section/heading.html" with title=title|default:_("Untitled section") %} {% endslot %}
{% slot content %} diff --git a/hexa/ui/templates/ui/section/section_heading_with_label.html b/hexa/ui/templates/ui/section/section_heading_with_label.html deleted file mode 100644 index c75899514..000000000 --- a/hexa/ui/templates/ui/section/section_heading_with_label.html +++ /dev/null @@ -1,7 +0,0 @@ -{# https://tailwindui.com/components/application-ui/headings/section-headings#component-446e3bebd49060a01c3f05189c4c09da #} -
-

- {{ title }} -

-

{{ label }}

-