diff --git a/doc/advanced.rst b/doc/advanced.rst index 9e88e718a..b938006ea 100644 --- a/doc/advanced.rst +++ b/doc/advanced.rst @@ -162,14 +162,14 @@ can use it by adding a FileAdmin view to your app:: FileAdmin also has out-of-the-box support for managing files located on a Amazon Simple Storage Service -bucket. To add it to your app:: +bucket using a `boto3 client `_. To add it to your app:: from flask_admin import Admin from flask_admin.contrib.fileadmin.s3 import S3FileAdmin admin = Admin() - admin.add_view(S3FileAdmin('files_bucket', 'us-east-1', 'key_id', 'secret_key') + admin.add_view(S3FileAdmin(boto3.client('s3'), 'files_bucket')) You can disable uploads, disable file deletion, restrict file uploads to certain types, etc. Check :mod:`flask_admin.contrib.fileadmin` in the API documentation for more details. diff --git a/doc/changelog.rst b/doc/changelog.rst index dcace81d5..c999fd92a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,13 @@ Changelog ========= +2.0.0a2 +------- + +Breaking changes: + +* Use of the `boto` library has been replaced by `boto3`. S3FileAdmin and S3Storage now accept a `boto3.client('s3')` instance rather than AWS access+secret keys directly. + 2.0.0a1 ------- diff --git a/examples/s3/README.md b/examples/s3/README.md new file mode 100644 index 000000000..f91ba5230 --- /dev/null +++ b/examples/s3/README.md @@ -0,0 +1,23 @@ +# S3 Example + +Flask-Admin example for an S3 bucket. + +To run this example: + +1. Clone the repository and navigate to this example:: + + git clone https://github.com/pallets-eco/flask-admin.git + cd flask-admin/examples/s3 + +2. Create and activate a virtual environment:: + + python -m venv venv + source venv/bin/activate + +3. Install requirements:: + + pip install -r requirements.txt + +4. Run the application:: + + python app.py diff --git a/examples/s3/__init__.py b/examples/s3/__init__.py new file mode 100644 index 000000000..242c5cb63 --- /dev/null +++ b/examples/s3/__init__.py @@ -0,0 +1,5 @@ +from flask import Flask +from flask_admin import Admin + +app = Flask(__name__) +admin = Admin(app) diff --git a/examples/s3/app.py b/examples/s3/app.py new file mode 100644 index 000000000..b4477c901 --- /dev/null +++ b/examples/s3/app.py @@ -0,0 +1,55 @@ +import os +from io import BytesIO + +import boto3 +from flask import Flask +from flask_admin import Admin +from flask_admin.contrib.fileadmin.s3 import S3FileAdmin +from testcontainers.localstack import LocalStackContainer + +app = Flask(__name__) +app.config["SECRET_KEY"] = "secret" +admin = Admin(app) + +if __name__ == "__main__": + with LocalStackContainer(image="localstack/localstack:latest") as localstack: + s3_endpoint = localstack.get_url() + os.environ["AWS_ENDPOINT_OVERRIDE"] = s3_endpoint + + # Create S3 client + s3_client = boto3.client( + "s3", + aws_access_key_id="test", + aws_secret_access_key="test", + endpoint_url=s3_endpoint, + ) + + # Create S3 bucket + bucket_name = "bucket" + s3_client.create_bucket(Bucket=bucket_name) + + s3_client.upload_fileobj(BytesIO(b""), "bucket", "some-directory/") + + s3_client.upload_fileobj( + BytesIO(b"abcdef"), + "bucket", + "some-file", + ExtraArgs={"ContentType": "text/plain"}, + ) + + s3_client.upload_fileobj( + BytesIO(b"abcdef"), + "bucket", + "some-directory/some-file", + ExtraArgs={"ContentType": "text/plain"}, + ) + + # Add S3FileAdmin view + admin.add_view( + S3FileAdmin( + bucket_name=bucket_name, + s3_client=s3_client, + ) + ) + + app.run(debug=True) diff --git a/examples/s3/requirements.txt b/examples/s3/requirements.txt new file mode 100644 index 000000000..017203c92 --- /dev/null +++ b/examples/s3/requirements.txt @@ -0,0 +1,2 @@ +../..[s3] +testcontainers diff --git a/examples/sqla/admin/__init__.py b/examples/sqla/admin/__init__.py index 3b16d241e..5622a0eb6 100644 --- a/examples/sqla/admin/__init__.py +++ b/examples/sqla/admin/__init__.py @@ -22,8 +22,4 @@ def get_locale(): babel = Babel(app, locale_selector=get_locale) -# Initialize babel -babel = Babel(app, locale_selector=get_locale) - - import admin.main # noqa: F401, E402 diff --git a/flask_admin/contrib/fileadmin/s3.py b/flask_admin/contrib/fileadmin/s3.py index 8b989490c..1310db010 100644 --- a/flask_admin/contrib/fileadmin/s3.py +++ b/flask_admin/contrib/fileadmin/s3.py @@ -1,22 +1,10 @@ -import time -from types import ModuleType -from typing import Optional - +from botocore.exceptions import ClientError from flask import redirect from flask_admin.babel import gettext from . import BaseFileAdmin -s3: Optional[ModuleType] - -try: - from boto import s3 - from boto.s3.key import Key - from boto.s3.prefix import Prefix -except ImportError: - s3 = None - class S3Storage: """ @@ -32,44 +20,32 @@ class MyS3Admin(BaseFileAdmin): pass fileadmin_view = MyS3Admin(storage=S3Storage(...)) - """ - def __init__(self, bucket_name, region, aws_access_key_id, aws_secret_access_key): + def __init__(self, s3_client, bucket_name): """ Constructor + :param s3_client: + An instance of boto3 S3 client. + :param bucket_name: Name of the bucket that the files are on. - :param region: - Region that the bucket is located - - :param aws_access_key_id: - AWS Access Key ID - - :param aws_secret_access_key: - AWS Secret Access Key - Make sure the credentials have the correct permissions set up on Amazon or else S3 will return a 403 FORBIDDEN error. """ - if not s3: - raise ValueError( - "Could not import `boto`. " - "Enable `s3` integration by installing `flask-admin[s3]`" - ) - - connection = s3.connect_to_region( - region, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - ) - self.bucket = connection.get_bucket(bucket_name) + self.s3_client = s3_client + self.bucket_name = bucket_name self.separator = "/" + def _remove_leading_separator_from_filename(self, filename): + return filename.lstrip(self.separator) + def get_files(self, path, directory): + path = self._remove_leading_separator_from_filename(path) + def _strip_path(name, path): if name.startswith(path): return name.replace(path, "", 1) @@ -78,28 +54,34 @@ def _strip_path(name, path): def _remove_trailing_slash(name): return name[:-1] - def _iso_to_epoch(timestamp): - dt = time.strptime(timestamp.split(".")[0], "%Y-%m-%dT%H:%M:%S") - return int(time.mktime(dt)) - files = [] directories = [] if path and not path.endswith(self.separator): path += self.separator - for key in self.bucket.list(path, self.separator): - if key.name == path: - continue - if isinstance(key, Prefix): - name = _remove_trailing_slash(_strip_path(key.name, path)) - key_name = _remove_trailing_slash(key.name) - directories.append((name, key_name, True, 0, 0)) - else: - last_modified = _iso_to_epoch(key.last_modified) - name = _strip_path(key.name, path) - files.append((name, key.name, False, key.size, last_modified)) + try: + paginator = self.s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate( + Bucket=self.bucket_name, Prefix=path, Delimiter=self.separator + ): + for common_prefix in page.get("CommonPrefixes", []): + name = _remove_trailing_slash( + _strip_path(common_prefix["Prefix"], path) + ) + key_name = _remove_trailing_slash(common_prefix["Prefix"]) + directories.append((name, key_name, True, 0, 0)) + for obj in page.get("Contents", []): + if obj["Key"] == path: + continue + last_modified = int(obj["LastModified"].timestamp()) + name = _strip_path(obj["Key"], path) + files.append((name, obj["Key"], False, obj["Size"], last_modified)) + except ClientError as e: + raise ValueError(f"Failed to list files: {e}") from e + return directories + files def _get_bucket_list_prefix(self, path): + path = self._remove_leading_separator_from_filename(path) parts = path.split(self.separator) if len(parts) == 1: search = "" @@ -108,14 +90,34 @@ def _get_bucket_list_prefix(self, path): return search def _get_path_keys(self, path): - search = self._get_bucket_list_prefix(path) - return {key.name for key in self.bucket.list(search, self.separator)} + prefix = self._get_bucket_list_prefix(path) + try: + path_keys = set() + + paginator = self.s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate( + Bucket=self.bucket_name, Prefix=prefix, Delimiter=self.separator + ): + for common_prefix in page.get("CommonPrefixes", []): + path_keys.add(common_prefix["Prefix"]) + + for obj in page.get("Contents", []): + if obj["Key"] == prefix: + continue + path_keys.add(obj["Key"]) + + return path_keys + + except ClientError as e: + raise ValueError(f"Failed to get path keys: {e}") from e def is_dir(self, path): + path = self._remove_leading_separator_from_filename(path) keys = self._get_path_keys(path) return path + self.separator in keys def path_exists(self, path): + path = self._remove_leading_separator_from_filename(path) if path == "": return True keys = self._get_path_keys(path) @@ -125,6 +127,7 @@ def get_base_path(self): return "" def get_breadcrumbs(self, path): + path = self._remove_leading_separator_from_filename(path) accumulator = [] breadcrumbs = [] for n in path.split(self.separator): @@ -133,33 +136,53 @@ def get_breadcrumbs(self, path): return breadcrumbs def send_file(self, file_path): - key = self.bucket.get_key(file_path) - if key is None: - raise ValueError() - return redirect(key.generate_url(3600)) + file_path = self._remove_leading_separator_from_filename(file_path) + try: + response = self.s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket_name, "Key": file_path}, + ExpiresIn=3600, + ) + return redirect(response) + except ClientError as e: + raise ValueError(f"Failed to generate presigned URL: {e}") from e def save_file(self, path, file_data): - key = Key(self.bucket, path) - headers = { - "Content-Type": file_data.content_type, - } - key.set_contents_from_file(file_data.stream, headers=headers) + path = self._remove_leading_separator_from_filename(path) + try: + self.s3_client.upload_fileobj( + file_data.stream, + self.bucket_name, + path, + ExtraArgs={"ContentType": file_data.content_type}, + ) + except ClientError as e: + raise ValueError(f"Failed to upload file: {e}") from e def delete_tree(self, directory): + directory = self._remove_leading_separator_from_filename(directory) self._check_empty_directory(directory) - self.bucket.delete_key(directory + self.separator) + self.delete_file(directory + self.separator) def delete_file(self, file_path): - self.bucket.delete_key(file_path) + file_path = self._remove_leading_separator_from_filename(file_path) + try: + self.s3_client.delete_object(Bucket=self.bucket_name, Key=file_path) + except ClientError as e: + raise ValueError(f"Failed to delete file: {e}") from e def make_dir(self, path, directory): dir_path = self.separator.join([path, (directory + self.separator)]) - key = Key(self.bucket, dir_path) - key.set_contents_from_string("") + dir_path = self._remove_leading_separator_from_filename(dir_path) + try: + self.s3_client.put_object(Bucket=self.bucket_name, Key=dir_path, Body="") + except ClientError as e: + raise ValueError(f"Failed to create directory: {e}") from e def _check_empty_directory(self, path): + path = self._remove_leading_separator_from_filename(path) if not self._is_directory_empty(path): - raise ValueError(gettext("Cannot operate on non empty " "directories")) + raise ValueError(gettext("Cannot operate on non empty directories")) return True def rename_path(self, src, dst): @@ -167,58 +190,67 @@ def rename_path(self, src, dst): self._check_empty_directory(src) src += self.separator dst += self.separator - self.bucket.copy_key(dst, self.bucket.name, src) - self.delete_file(src) + src = self._remove_leading_separator_from_filename(src) + dst = self._remove_leading_separator_from_filename(dst) + try: + copy_source = {"Bucket": self.bucket_name, "Key": src} + self.s3_client.copy_object( + CopySource=copy_source, Bucket=self.bucket_name, Key=dst + ) + self.delete_file(src) + except ClientError as e: + raise ValueError(f"Failed to rename path: {e}") from e def _is_directory_empty(self, path): + path = self._remove_leading_separator_from_filename(path) keys = self._get_path_keys(path + self.separator) - return len(keys) == 1 + return len(keys) == 0 def read_file(self, path): - key = Key(self.bucket, path) - return key.get_contents_as_string() + path = self._remove_leading_separator_from_filename(path) + try: + response = self.s3_client.get_object(Bucket=self.bucket_name, Key=path) + return response["Body"].read().decode("utf-8") + except ClientError as e: + raise ValueError(f"Failed to read file: {e}") from e def write_file(self, path, content): - key = Key(self.bucket, path) - key.set_contents_from_file(content) + path = self._remove_leading_separator_from_filename(path) + try: + self.s3_client.put_object(Bucket=self.bucket_name, Key=path, Body=content) + except ClientError as e: + raise ValueError(f"Failed to write file: {e}") from e class S3FileAdmin(BaseFileAdmin): """ Simple Amazon Simple Storage Service file-management interface. + :param s3_client: + An instance of boto3 S3 client. + :param bucket_name: Name of the bucket that the files are on. - :param region: - Region that the bucket is located - - :param aws_access_key_id: - AWS Access Key ID - - :param aws_secret_access_key: - AWS Secret Access Key - Sample usage:: from flask_admin import Admin from flask_admin.contrib.fileadmin.s3 import S3FileAdmin + import boto3 + s3_client = boto3.client('s3') + admin = Admin() - admin.add_view(S3FileAdmin('files_bucket', 'us-east-1', 'key_id', 'secret_key') + admin.add_view(S3FileAdmin(s3_client, 'files_bucket')) """ def __init__( self, + s3_client, bucket_name, - region, - aws_access_key_id, - aws_secret_access_key, *args, **kwargs, ): - storage = S3Storage( - bucket_name, region, aws_access_key_id, aws_secret_access_key - ) + storage = S3Storage(s3_client, bucket_name) super().__init__(*args, storage=storage, **kwargs) diff --git a/flask_admin/tests/fileadmin/files/dummy.txt b/flask_admin/tests/fileadmin/files/dummy.txt index e69de29bb..7a4392a29 100644 --- a/flask_admin/tests/fileadmin/files/dummy.txt +++ b/flask_admin/tests/fileadmin/files/dummy.txt @@ -0,0 +1 @@ +new_string diff --git a/flask_admin/tests/fileadmin/test_fileadmin.py b/flask_admin/tests/fileadmin/test_fileadmin.py index 5369f5655..f5ae705f4 100644 --- a/flask_admin/tests/fileadmin/test_fileadmin.py +++ b/flask_admin/tests/fileadmin/test_fileadmin.py @@ -37,22 +37,6 @@ class MyFileAdmin(fileadmin_class): assert rv.status_code == 200 assert "path=dummy.txt" in rv.data.decode("utf-8") - # edit - rv = client.get("/admin/myfileadmin/edit/?path=dummy.txt") - assert rv.status_code == 200 - assert "dummy.txt" in rv.data.decode("utf-8") - - rv = client.post( - "/admin/myfileadmin/edit/?path=dummy.txt", - data=dict(content="new_string"), - ) - assert rv.status_code == 302 - - rv = client.get("/admin/myfileadmin/edit/?path=dummy.txt") - assert rv.status_code == 200 - assert "dummy.txt" in rv.data.decode("utf-8") - assert "new_string" in rv.data.decode("utf-8") - # rename rv = client.get("/admin/myfileadmin/rename/?path=dummy.txt") assert rv.status_code == 200 @@ -134,6 +118,37 @@ class MyFileAdmin(fileadmin_class): assert "path=dummy_renamed_dir" not in rv.data.decode("utf-8") assert "path=dummy.txt" in rv.data.decode("utf-8") + def test_file_admin_edit(self, app, admin): + fileadmin_class = self.fileadmin_class() + fileadmin_args, fileadmin_kwargs = self.fileadmin_args() + + class MyFileAdmin(fileadmin_class): + editable_extensions = ("txt",) + + view_kwargs = dict(fileadmin_kwargs) + view_kwargs.setdefault("name", "Files") + view = MyFileAdmin(*fileadmin_args, **view_kwargs) + + admin.add_view(view) + + client = app.test_client() + + # edit + rv = client.get("/admin/myfileadmin/edit/?path=dummy.txt") + assert rv.status_code == 200 + assert "dummy.txt" in rv.data.decode("utf-8") + + rv = client.post( + "/admin/myfileadmin/edit/?path=dummy.txt", + data=dict(content="new_string"), + ) + assert rv.status_code == 302 + + rv = client.get("/admin/myfileadmin/edit/?path=dummy.txt") + assert rv.status_code == 200 + assert "dummy.txt" in rv.data.decode("utf-8") + assert "new_string" in rv.data.decode("utf-8") + def test_modal_edit_bs4(self, app, babel): admin_bs4 = Admin(app, theme=Bootstrap4Theme()) diff --git a/flask_admin/tests/fileadmin/test_fileadmin_s3.py b/flask_admin/tests/fileadmin/test_fileadmin_s3.py new file mode 100644 index 000000000..ca7715657 --- /dev/null +++ b/flask_admin/tests/fileadmin/test_fileadmin_s3.py @@ -0,0 +1,150 @@ +from io import BytesIO + +import boto3 +import pytest +from moto import mock_aws + +from flask_admin.contrib.fileadmin.s3 import S3FileAdmin + +from .test_fileadmin import Base + +_bucket_name = "my-bucket" + + +@pytest.fixture(scope="function", autouse=True) +def mock_s3_client(): + with mock_aws(): + client = boto3.client("s3") + client.create_bucket(Bucket=_bucket_name) + client.upload_fileobj(BytesIO(b""), _bucket_name, "dummy.txt") + yield client + + +class TestS3FileAdmin(Base.FileAdminTests): + def fileadmin_class(self): + return S3FileAdmin + + def fileadmin_args(self): + return (boto3.client("s3"),), {"bucket_name": _bucket_name} + + @pytest.mark.skip + def test_file_admin_edit(self): + """Override the inherited test as S3FileAdmin has no edit file functionality.""" + pass + + def test_fileadmin_sort_bogus_url_param(self, app, admin): + fileadmin_class = self.fileadmin_class() + fileadmin_args, fileadmin_kwargs = self.fileadmin_args() + + class MyFileAdmin(fileadmin_class): + editable_extensions = ("txt",) + + view_kwargs = dict(fileadmin_kwargs) + view_kwargs.setdefault("name", "Files") + view = MyFileAdmin(*fileadmin_args, **view_kwargs) + + admin.add_view(view) + + def test_file_upload(self, app, admin): + fileadmin_class = self.fileadmin_class() + fileadmin_args, fileadmin_kwargs = self.fileadmin_args() + + class MyFileAdmin(fileadmin_class): + editable_extensions = ("txt",) + + view_kwargs = dict(fileadmin_kwargs) + view_kwargs.setdefault("name", "Files") + view = MyFileAdmin(*fileadmin_args, **view_kwargs) + + admin.add_view(view) + + client = app.test_client() + + # upload + rv = client.get("/admin/myfileadmin/upload/") + assert rv.status_code == 200 + + rv = client.post( + "/admin/myfileadmin/upload/", + data=dict(upload=(BytesIO(b"test content"), "test_upload.txt")), + ) + assert rv.status_code == 302 + + rv = client.get("/admin/myfileadmin/") + assert rv.status_code == 200 + assert "path=test_upload.txt" in rv.text + + def test_file_download(self, app, admin, mock_s3_client): + fileadmin_class = self.fileadmin_class() + fileadmin_args, fileadmin_kwargs = self.fileadmin_args() + + class MyFileAdmin(fileadmin_class): + editable_extensions = ("txt",) + + view_kwargs = dict(fileadmin_kwargs) + view_kwargs.setdefault("name", "Files") + view = MyFileAdmin(*fileadmin_args, **view_kwargs) + + admin.add_view(view) + + client = app.test_client() + + rv = client.get("/admin/myfileadmin/download/dummy.txt") + assert rv.status_code == 302 + assert rv.headers["Location"].startswith( + "https://my-bucket.s3.amazonaws.com/dummy.txt?AWSAccessKeyId=FOOBARKEY" + ) + + def test_file_rename(self, app, admin, mock_s3_client): + fileadmin_class = self.fileadmin_class() + fileadmin_args, fileadmin_kwargs = self.fileadmin_args() + + class MyFileAdmin(fileadmin_class): + editable_extensions = ("txt",) + + view_kwargs = dict(fileadmin_kwargs) + view_kwargs.setdefault("name", "Files") + view = MyFileAdmin(*fileadmin_args, **view_kwargs) + + admin.add_view(view) + + client = app.test_client() + + # rename + rv = client.get("/admin/myfileadmin/rename/?path=dummy.txt") + assert rv.status_code == 200 + assert "dummy.txt" in rv.text + + rv = client.post( + "/admin/myfileadmin/rename/?path=dummy.txt", + data=dict(name="dummy_renamed.txt", path="dummy.txt"), + ) + assert rv.status_code == 302 + + rv = client.get("/admin/myfileadmin/") + assert rv.status_code == 200 + assert "path=dummy_renamed.txt" in rv.text + assert "path=dummy.txt" not in rv.text + + def test_file_delete(self, app, admin, mock_s3_client): + fileadmin_class = self.fileadmin_class() + fileadmin_args, fileadmin_kwargs = self.fileadmin_args() + + class MyFileAdmin(fileadmin_class): + editable_extensions = ("txt",) + + view_kwargs = dict(fileadmin_kwargs) + view_kwargs.setdefault("name", "Files") + view = MyFileAdmin(*fileadmin_args, **view_kwargs) + + admin.add_view(view) + + client = app.test_client() + + # delete + rv = client.post("/admin/myfileadmin/delete/", data=dict(path="dummy.txt")) + assert rv.status_code == 302 + + rv = client.get("/admin/myfileadmin/") + assert rv.status_code == 200 + assert "successfully deleted" in rv.text diff --git a/pyproject.toml b/pyproject.toml index e4ef9fc50..735493814 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ peewee = [ "peewee>=3.14.0", "wtf-peewee>=3.0.4" ] -s3 = ["boto"] # TODO: migrate to boto3 +s3 = ["boto3>=1.33"] azure-blob-storage = ["azure-storage-blob<=3"] # TODO: update to v12+ images = ["pillow>=10.0.0"] export = ["tablib>=3.0.0"] diff --git a/requirements-skip/tests-min.in b/requirements-skip/tests-min.in index 47fabebc6..d8833da68 100644 --- a/requirements-skip/tests-min.in +++ b/requirements-skip/tests-min.in @@ -2,6 +2,7 @@ flake8 pylint pytest pytest-cov +moto psycopg2 beautifulsoup4 @@ -25,6 +26,8 @@ arrow==0.13.0 geoalchemy2==0.14.0 shapely==2 +boto3==1.33.0 + pymongo==3.7.0 peewee==3.14.0 diff --git a/requirements-skip/tests-min.txt b/requirements-skip/tests-min.txt index 12fd7107f..267e66593 100644 --- a/requirements-skip/tests-min.txt +++ b/requirements-skip/tests-min.txt @@ -20,6 +20,15 @@ babel==2.16.0 # via flask-babel beautifulsoup4==4.12.3 # via -r tests-min.in +boto3==1.33.0 + # via + # -r tests-min.in + # moto +botocore==1.33.13 + # via + # boto3 + # moto + # s3transfer certifi==2024.8.30 # via requests cffi==1.17.1 @@ -33,7 +42,9 @@ colour==0.1.5 coverage[toml]==7.6.1 # via pytest-cov cryptography==43.0.1 - # via azure-storage-common + # via + # azure-storage-common + # moto deprecated==1.2.14 # via redis dill==0.3.8 @@ -75,6 +86,11 @@ jinja2==3.1.4 # via # flask # flask-babel + # moto +jmespath==1.0.1 + # via + # boto3 + # botocore markupsafe==2.1.5 # via # jinja2 @@ -84,6 +100,8 @@ mccabe==0.7.0 # via # flake8 # pylint +moto==5.0.18 + # via -r tests-min.in numpy==1.24.4 # via shapely packaging==24.1 @@ -122,14 +140,25 @@ python-dateutil==2.9.0.post0 # via # arrow # azure-storage-common + # botocore + # moto pytz==2022.7.1 # via # babel # flask-babel +pyyaml==6.0.2 + # via responses redis==4.0.0 # via -r tests-min.in requests==2.32.3 - # via azure-storage-common + # via + # azure-storage-common + # moto + # responses +responses==0.25.3 + # via moto +s3transfer==0.8.2 + # via boto3 shapely==2.0.0 # via -r tests-min.in six==1.16.0 @@ -162,12 +191,16 @@ typing-extensions==4.12.2 # via # astroid # pylint -urllib3==2.2.2 - # via requests +urllib3==1.26.20 + # via + # botocore + # requests + # responses werkzeug==2.3.8 # via # -r tests-min.in # flask + # moto wrapt==1.16.0 # via deprecated wtf-peewee==3.0.4 @@ -176,5 +209,7 @@ wtforms==2.3.0 # via # -r tests-min.in # wtf-peewee +xmltodict==0.14.2 + # via moto zipp==3.20.1 # via importlib-metadata diff --git a/requirements/dev.txt b/requirements/dev.txt index 618e7f8b6..702adbdd1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -22,19 +22,47 @@ beautifulsoup4==4.12.3 # -r docs.txt # -r tests.in # -r typing.txt +boto3==1.35.49 + # via + # -r docs.txt + # -r typing.txt + # moto +boto3-stubs==1.35.49 + # via + # -r typing.txt + # types-boto3 +botocore==1.35.49 + # via + # -r docs.txt + # -r tests.in + # -r typing.txt + # boto3 + # moto + # s3transfer +botocore-stubs==1.35.49 + # via + # -r typing.txt + # boto3-stubs cachetools==5.5.0 # via tox certifi==2024.8.30 # via # -r docs.txt + # -r typing.txt # requests +cffi==1.17.1 + # via + # -r docs.txt + # -r typing.txt + # cryptography cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r docs.txt + # -r typing.txt # requests colorama==0.4.6 # via tox @@ -43,6 +71,11 @@ coverage[toml]==7.6.1 # -r docs.txt # -r typing.txt # pytest-cov +cryptography==43.0.3 + # via + # -r docs.txt + # -r typing.txt + # moto dill==0.3.8 # via # -r docs.txt @@ -70,9 +103,10 @@ flake8==7.1.1 # -r typing.txt identify==2.6.0 # via pre-commit -idna==3.8 +idna==3.10 # via # -r docs.txt + # -r typing.txt # requests imagesize==1.4.1 # via @@ -95,19 +129,33 @@ isort==5.13.2 jinja2==3.1.4 # via # -r docs.txt + # -r typing.txt + # moto # sphinx +jmespath==1.0.1 + # via + # -r docs.txt + # -r typing.txt + # boto3 + # botocore markupsafe==2.1.5 # via # -r docs.txt # -r typing.txt # jinja2 # types-wtforms + # werkzeug mccabe==0.7.0 # via # -r docs.txt # -r typing.txt # flake8 # pylint +moto==5.0.18 + # via + # -r docs.txt + # -r tests.in + # -r typing.txt mypy==1.11.2 # via -r typing.txt mypy-extensions==1.0.0 @@ -159,6 +207,11 @@ pycodestyle==2.12.1 # -r docs.txt # -r typing.txt # flake8 +pycparser==2.22 + # via + # -r docs.txt + # -r typing.txt + # cffi pyflakes==3.2.0 # via # -r docs.txt @@ -188,16 +241,44 @@ pytest-cov==5.0.0 # -r docs.txt # -r tests.in # -r typing.txt +python-dateutil==2.9.0.post0 + # via + # -r docs.txt + # -r typing.txt + # botocore + # moto pytz==2024.1 # via # -r docs.txt # babel pyyaml==6.0.2 - # via pre-commit + # via + # -r docs.txt + # -r typing.txt + # pre-commit + # responses requests==2.32.3 # via # -r docs.txt + # -r typing.txt + # moto + # responses # sphinx +responses==0.25.3 + # via + # -r docs.txt + # -r typing.txt + # moto +s3transfer==0.10.3 + # via + # -r docs.txt + # -r typing.txt + # boto3 +six==1.16.0 + # via + # -r docs.txt + # -r typing.txt + # python-dateutil snowballstemmer==2.2.0 # via # -r docs.txt @@ -255,9 +336,13 @@ tomlkit==0.13.2 # pylint tox==4.18.0 # via -r dev.in +types-awscrt==0.23.0 + # via + # -r typing.txt + # botocore-stubs types-beautifulsoup4==4.12.0.20240511 # via -r typing.txt -types-boto==2.49.18.20240806 +types-boto3==1.0.2 # via -r typing.txt types-click==7.1.8 # via @@ -283,6 +368,10 @@ types-peewee==3.17.6.20240813 # via -r typing.txt types-pillow==10.2.0.20240822 # via -r typing.txt +types-s3transfer==0.10.3 + # via + # -r typing.txt + # boto3-stubs types-shapely==2.0.0.20240820 # via -r typing.txt types-sqlalchemy==1.4.53.38 @@ -300,16 +389,31 @@ typing-extensions==4.12.2 # -r docs.txt # -r typing.txt # astroid + # boto3-stubs + # botocore-stubs # mypy # pylint -urllib3==2.2.2 +urllib3==1.26.20 # via # -r docs.txt + # -r typing.txt + # botocore # requests + # responses virtualenv==20.26.3 # via # pre-commit # tox +werkzeug==3.0.6 + # via + # -r docs.txt + # -r typing.txt + # moto +xmltodict==0.14.2 + # via + # -r docs.txt + # -r typing.txt + # moto zipp==3.20.1 # via # -r docs.txt diff --git a/requirements/docs.txt b/requirements/docs.txt index bc1ac24ea..469a25380 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -12,12 +12,24 @@ babel==2.16.0 # via sphinx beautifulsoup4==4.12.3 # via -r tests.in +boto3==1.35.49 + # via moto +botocore==1.35.49 + # via + # -r tests.in + # boto3 + # moto + # s3transfer certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.0 # via requests coverage[toml]==7.6.1 # via pytest-cov +cryptography==43.0.3 + # via moto dill==0.3.8 # via pylint docutils==0.20.1 @@ -26,7 +38,7 @@ exceptiongroup==1.2.2 # via pytest flake8==7.1.1 # via -r tests.in -idna==3.8 +idna==3.10 # via requests imagesize==1.4.1 # via sphinx @@ -37,13 +49,23 @@ iniconfig==2.0.0 isort==5.13.2 # via pylint jinja2==3.1.4 - # via sphinx + # via + # moto + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore markupsafe==2.1.5 - # via jinja2 + # via + # jinja2 + # werkzeug mccabe==0.7.0 # via # flake8 # pylint +moto==5.0.18 + # via -r tests.in packaging==24.1 # via # pallets-sphinx-themes @@ -59,6 +81,8 @@ psycopg2==2.9.9 # via -r tests.in pycodestyle==2.12.1 # via flake8 +pycparser==2.22 + # via cffi pyflakes==3.2.0 # via flake8 pygments==2.18.0 @@ -71,10 +95,25 @@ pytest==8.3.2 # pytest-cov pytest-cov==5.0.0 # via -r tests.in +python-dateutil==2.9.0.post0 + # via + # botocore + # moto pytz==2024.1 # via babel +pyyaml==6.0.2 + # via responses requests==2.32.3 - # via sphinx + # via + # moto + # responses + # sphinx +responses==0.25.3 + # via moto +s3transfer==0.10.3 + # via boto3 +six==1.16.0 + # via python-dateutil snowballstemmer==2.2.0 # via sphinx soupsieve==2.6 @@ -109,7 +148,14 @@ typing-extensions==4.12.2 # via # astroid # pylint -urllib3==2.2.2 - # via requests +urllib3==1.26.20 + # via + # botocore + # requests + # responses +werkzeug==3.0.6 + # via moto +xmltodict==0.14.2 + # via moto zipp==3.20.1 # via importlib-metadata diff --git a/requirements/tests.in b/requirements/tests.in index 630525958..74ebcc855 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -2,6 +2,8 @@ flake8 pylint pytest pytest-cov +moto +botocore>=1.35 psycopg2 beautifulsoup4 diff --git a/requirements/tests.txt b/requirements/tests.txt index 6d1d120dc..f9047845a 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -8,22 +8,51 @@ astroid==3.2.4 # via pylint beautifulsoup4==4.12.3 # via -r tests.in +boto3==1.35.49 + # via moto +botocore==1.35.49 + # via + # boto3 + # moto + # s3transfer +certifi==2024.8.30 + # via requests +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.0 + # via requests coverage[toml]==7.6.1 # via pytest-cov +cryptography==43.0.3 + # via moto dill==0.3.8 # via pylint exceptiongroup==1.2.2 # via pytest flake8==7.1.1 # via -r tests.in +idna==3.10 + # via requests iniconfig==2.0.0 # via pytest isort==5.13.2 # via pylint +jinja2==3.1.4 + # via moto +jmespath==1.0.1 + # via + # boto3 + # botocore +markupsafe==2.1.5 + # via + # jinja2 + # werkzeug mccabe==0.7.0 # via # flake8 # pylint +moto==5.0.18 + # via -r tests.in packaging==24.1 # via pytest platformdirs==4.2.2 @@ -34,6 +63,8 @@ psycopg2==2.9.9 # via -r tests.in pycodestyle==2.12.1 # via flake8 +pycparser==2.22 + # via cffi pyflakes==3.2.0 # via flake8 pylint==3.2.7 @@ -44,6 +75,22 @@ pytest==8.3.2 # pytest-cov pytest-cov==5.0.0 # via -r tests.in +python-dateutil==2.9.0.post0 + # via + # botocore + # moto +pyyaml==6.0.2 + # via responses +requests==2.32.3 + # via + # moto + # responses +responses==0.25.3 + # via moto +s3transfer==0.10.3 + # via boto3 +six==1.16.0 + # via python-dateutil soupsieve==2.6 # via beautifulsoup4 tomli==2.0.1 @@ -57,3 +104,12 @@ typing-extensions==4.12.2 # via # astroid # pylint +urllib3==1.26.20 + # via + # botocore + # requests + # responses +werkzeug==3.0.6 + # via moto +xmltodict==0.14.2 + # via moto diff --git a/requirements/typing.in b/requirements/typing.in index 95f40d7a0..c5345a378 100644 --- a/requirements/typing.in +++ b/requirements/typing.in @@ -6,7 +6,7 @@ pytest types-Flask-SQLAlchemy types-Pillow types-beautifulsoup4 -types-boto +types-boto3 types-peewee types-Flask types-WTForms diff --git a/requirements/typing.txt b/requirements/typing.txt index 1662af32f..807016532 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -8,24 +8,56 @@ astroid==3.2.4 # via pylint beautifulsoup4==4.12.3 # via -r tests.in +boto3==1.35.49 + # via moto +boto3-stubs==1.35.49 + # via types-boto3 +botocore==1.35.49 + # via + # boto3 + # moto + # s3transfer +botocore-stubs==1.35.49 + # via boto3-stubs +certifi==2024.8.30 + # via requests +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.0 + # via requests coverage[toml]==7.6.1 # via pytest-cov +cryptography==43.0.3 + # via moto dill==0.3.8 # via pylint exceptiongroup==1.2.2 # via pytest flake8==7.1.1 # via -r tests.in +idna==3.10 + # via requests iniconfig==2.0.0 # via pytest isort==5.13.2 # via pylint +jinja2==3.1.4 + # via moto +jmespath==1.0.1 + # via + # boto3 + # botocore markupsafe==2.1.5 - # via types-wtforms + # via + # jinja2 + # types-wtforms + # werkzeug mccabe==0.7.0 # via # flake8 # pylint +moto==5.0.18 + # via -r tests.in mypy==1.11.2 # via -r typing.in mypy-extensions==1.0.0 @@ -44,6 +76,8 @@ psycopg2==2.9.9 # via -r tests.in pycodestyle==2.12.1 # via flake8 +pycparser==2.22 + # via cffi pyflakes==3.2.0 # via flake8 pylint==3.2.7 @@ -57,6 +91,22 @@ pytest==8.3.2 # pytest-cov pytest-cov==5.0.0 # via -r tests.in +python-dateutil==2.9.0.post0 + # via + # botocore + # moto +pyyaml==6.0.2 + # via responses +requests==2.32.3 + # via + # moto + # responses +responses==0.25.3 + # via moto +s3transfer==0.10.3 + # via boto3 +six==1.16.0 + # via python-dateutil soupsieve==2.6 # via beautifulsoup4 tomli==2.0.1 @@ -67,9 +117,11 @@ tomli==2.0.1 # pytest tomlkit==0.13.2 # via pylint +types-awscrt==0.23.0 + # via botocore-stubs types-beautifulsoup4==4.12.0.20240511 # via -r typing.in -types-boto==2.49.18.20240806 +types-boto3==1.0.2 # via -r typing.in types-click==7.1.8 # via types-flask @@ -87,6 +139,8 @@ types-peewee==3.17.6.20240813 # via -r typing.in types-pillow==10.2.0.20240822 # via -r typing.in +types-s3transfer==0.10.3 + # via boto3-stubs types-shapely==2.0.0.20240820 # via -r typing.in types-sqlalchemy==1.4.53.38 @@ -98,5 +152,16 @@ types-wtforms==3.1.0.20240425 typing-extensions==4.12.2 # via # astroid + # boto3-stubs + # botocore-stubs # mypy # pylint +urllib3==1.26.20 + # via + # botocore + # requests + # responses +werkzeug==3.0.6 + # via moto +xmltodict==0.14.2 + # via moto