diff --git a/tests/providers/dropbox/fixtures.py b/tests/providers/dropbox/fixtures.py index 318432212..78d3ad940 100644 --- a/tests/providers/dropbox/fixtures.py +++ b/tests/providers/dropbox/fixtures.py @@ -1,15 +1,31 @@ +import io import os import json import pytest -from tests.utils import MockStream +from waterbutler.core import streams +from waterbutler.providers.dropbox import DropboxProvider @pytest.fixture -def stream_10_MB(): - data = os.urandom(10000000) # 10 MB - return MockStream(data) +def auth(): + return {'name': 'cat', 'email': 'cat@cat.com'} + + +@pytest.fixture +def credentials(): + return {'token': 'wrote harry potter'} + + +@pytest.fixture +def other_credentials(): + return {'token': 'did not write harry potter'} + + +@pytest.fixture +def settings(): + return {'folder': '/Photos'} @pytest.fixture @@ -29,3 +45,28 @@ def revision_fixtures(): def error_fixtures(): with open(os.path.join(os.path.dirname(__file__), 'fixtures/errors.json'), 'r') as fp: return json.load(fp) + + +@pytest.fixture +def file_content(): + return b'SLEEP IS FOR THE WEAK GO SERVE STREAMS' + + +@pytest.fixture +def file_like(file_content): + return io.BytesIO(file_content) + + +@pytest.fixture +def file_stream(file_like): + return streams.FileStreamReader(file_like) + + +@pytest.fixture +def provider(auth, credentials, settings): + return DropboxProvider(auth, credentials, settings) + + +@pytest.fixture +def other_provider(auth, other_credentials, settings): + return DropboxProvider(auth, other_credentials, settings) diff --git a/tests/providers/dropbox/fixtures/root_provider.json b/tests/providers/dropbox/fixtures/root_provider.json index fbb8dce2f..40cbe165a 100644 --- a/tests/providers/dropbox/fixtures/root_provider.json +++ b/tests/providers/dropbox/fixtures/root_provider.json @@ -50,13 +50,13 @@ }, "property_groups":[ { - "template_id":"ptid:1a5n2i6d3OYEAAAAAAAAAYa", - "fields":[ - { - "name":"Security Policy", - "value":"Confidential" - } - ] + "template_id":"ptid:1a5n2i6d3OYEAAAAAAAAAYa", + "fields":[ + { + "name":"Security Policy", + "value":"Confidential" + } + ] } ] } @@ -127,34 +127,6 @@ "expires":"2047-08-22T15:34:03Z", "copy_reference":"AAAAAFuyfRE5aWhyemZmaTJzZXc" }, - "folder_with_hasmore_metadata":{ - "has_more":true, - "entries":[ - { - "name":"randomfolder", - "path_lower":"/conflict folder/randomfolder", - ".tag":"folder", - "path_display":"/conflict folder/randomfolder", - "id":"id:jki_ZJstdSAAAAAAAAAACw" - }, - { - "server_modified":"2017-08-25T17:14:41Z", - "path_lower":"/conflict folder/1dragon.rtf", - ".tag":"file", - "size":364, - "client_modified":"2017-08-25T17:14:41Z", - "name":"1dragon.rtf", - "content_hash":"060a25a56cfce28e8aefa8e32a55ff9e890cbd714cabb156e5feb41f9a7b1624", - "rev":"45bb27d11", - "path_display":"/conflict folder/1dragon.rtf", - "id":"id:jki_ZJstdSAAAAAAAAAACQ" - } - ], - "cursor":"AAHv3crtaRWT0SKHClfYwa4vmR72KjiW1wjLYSE3BLatCZSJ6pJnD4nWlmAXCZPDXGrxMaIZ0lrRMMAxACEmhJn47aYQCyb1pgOhYqWyinyADa99SpqL2gxdKsq-3I0nUrWd3AaGUiw6GzH5Lg8i_FYy2YyeLkETb7tjRTc6KUTjHCjD61T4A3ZYiNrDp_cSxWg" - }, - "session_metadata":{ - "session_id":"test session id" - }, "intra_move_copy_file_metadata_v2":{ "metadata":{ ".tag":"file", @@ -230,5 +202,8 @@ "name":"rename.rtf", "server_modified":"2017-08-29T15:37:04Z" } + }, + "session_metadata":{ + "session_id":"test session id" } } diff --git a/tests/providers/dropbox/test_provider.py b/tests/providers/dropbox/test_provider.py index 6e610a18f..cde754600 100644 --- a/tests/providers/dropbox/test_provider.py +++ b/tests/providers/dropbox/test_provider.py @@ -1,74 +1,34 @@ -import io import json from http import HTTPStatus import pytest import aiohttpretty -from waterbutler.core import streams from waterbutler.core.path import WaterButlerPath from waterbutler.core import metadata as core_metadata from waterbutler.core import exceptions as core_exceptions -from waterbutler.providers.dropbox import DropboxProvider +from waterbutler.providers.dropbox.settings import CHUNK_SIZE, CONTIGUOUS_UPLOAD_SIZE_LIMIT from waterbutler.providers.dropbox.metadata import (DropboxRevision, DropboxFileMetadata, DropboxFolderMetadata) from waterbutler.providers.dropbox.exceptions import (DropboxNamingConflictError, DropboxUnhandledConflictError) -from tests.providers.dropbox.fixtures import ( - stream_10_MB, - provider_fixtures, - revision_fixtures, - error_fixtures -) - -@pytest.fixture -def auth(): - return {'name': 'cat', 'email': 'cat@cat.com'} - - -@pytest.fixture -def credentials(): - return {'token': 'wrote harry potter'} - - -@pytest.fixture -def other_credentials(): - return {'token': 'did not write harry potter'} - - -@pytest.fixture -def settings(): - return {'folder': '/Photos'} - - -@pytest.fixture -def provider(auth, credentials, settings): - return DropboxProvider(auth, credentials, settings) - - -@pytest.fixture -def other_provider(auth, other_credentials, settings): - return DropboxProvider(auth, other_credentials, settings) - - -# file stream fixtures - -@pytest.fixture -def file_content(): - return b'SLEEP IS FOR THE WEAK GO SERVE STREAMS' - - -@pytest.fixture -def file_like(file_content): - return io.BytesIO(file_content) - - -@pytest.fixture -def file_stream(file_like): - return streams.FileStreamReader(file_like) +from tests.utils import MockCoroutine +from tests.providers.dropbox.fixtures import (auth, + settings, + provider, + file_like, + credentials, + file_stream, + file_content, + other_provider, + error_fixtures, + other_credentials, + provider_fixtures, + revision_fixtures, + ) def build_folder_metadata_data(path): @@ -247,7 +207,68 @@ async def test_upload(self, provider, provider_fixtures, error_fixtures, file_st @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_create_upload_session(self, provider, provider_fixtures): + async def test_upload_limit_contiguous_upload(self, provider, file_stream): + + assert file_stream.size == 38 + provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = 40 + + provider.metadata = MockCoroutine() + provider._contiguous_upload = MockCoroutine() + provider._chunked_upload = MockCoroutine() + + path = WaterButlerPath('/foobah') + await provider.upload(file_stream, path) + + assert provider._contiguous_upload.called_with(file_stream, path) + assert not provider._chunked_upload.called + + provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = CONTIGUOUS_UPLOAD_SIZE_LIMIT + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_upload_limit_chunked_upload(self, provider, provider_fixtures, file_stream): + + assert file_stream.size == 38 + provider.CHUNK_SIZE = 4 + provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = 15 + + provider.metadata = MockCoroutine() + provider._contiguous_upload = MockCoroutine() + provider._chunked_upload = MockCoroutine() + + path = WaterButlerPath('/foobah') + await provider.upload(file_stream, path) + + assert provider._chunked_upload.called_with(file_stream, path) + assert not provider._contiguous_upload.called + + provider.CHUNK_SIZE = CHUNK_SIZE + provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = CONTIGUOUS_UPLOAD_SIZE_LIMIT + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload(self, provider, file_stream, provider_fixtures): + + assert file_stream.size == 38 + provider.CHUNK_SIZE = 4 + + session_id = provider_fixtures.get('session_metadata').get('session_id', '') + provider._create_upload_session = MockCoroutine(return_value=session_id) + provider._upload_parts = MockCoroutine() + provider._complete_session = MockCoroutine() + + path = WaterButlerPath('/foobah') + await provider._chunked_upload(file_stream, path) + + assert provider._create_upload_session.called + assert provider._upload_parts.called_with(file_stream, session_id) + assert provider._complete_session.called_with(file_stream, path, session_id) + + provider.CHUNK_SIZE = CHUNK_SIZE + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload_create_upload_session(self, provider, provider_fixtures): url = provider._build_content_url('files', 'upload_session', 'start') @@ -255,49 +276,84 @@ async def test_create_upload_session(self, provider, provider_fixtures): 'POST', url, status=200, - body=provider_fixtures['session_metadata'] + body=provider_fixtures.get('session_metadata', '') ) session_id = await provider._create_upload_session() - assert session_id == 'test session id' + assert session_id == provider_fixtures.get('session_metadata').get('session_id', '') assert aiohttpretty.has_call(method='POST', uri=url) @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload_parts(self, provider, stream_10_MB): + async def test_chunked_upload_upload_parts(self, provider, file_stream, provider_fixtures): + + assert file_stream.size == 38 + provider.CHUNK_SIZE = 4 + + session_id = provider_fixtures.get('session_metadata').get('session_id', '') + provider._upload_part = MockCoroutine() + + await provider._upload_parts(file_stream, session_id) + assert provider._upload_part.call_count == 10 + + upload_args = { + 'close': False, + 'cursor': {'session_id': session_id, 'offset': 0, } + } + for i in range(0, 9): + upload_args['cursor']['offset'] = i * 4 + assert provider._upload_part.called_once_with(file_stream, provider.CHUNK_SIZE, upload_args) + upload_args['cursor']['offset'] = 36 + assert provider._upload_part.called_once_with(file_stream, 2, upload_args) + + provider.CHUNK_SIZE = CHUNK_SIZE + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload_upload_part(self, provider, file_stream, provider_fixtures): - url = provider._build_content_url('files', 'upload_session', 'append_v2') + assert file_stream.size == 38 + provider.CHUNK_SIZE = 4 + + session_id = provider_fixtures.get('session_metadata').get('session_id', '') + upload_args = { + 'close': False, + 'cursor': {'session_id': session_id, 'offset': 20, } + } - aiohttpretty.register_json_uri('POST', url, status=200) - await provider._upload_parts(stream_10_MB, 'session id') + upload_part_url = provider._build_content_url('files', 'upload_session', 'append_v2') + aiohttpretty.register_json_uri('POST', upload_part_url, status=200) - assert len(aiohttpretty.calls) == 3 - for call in aiohttpretty.calls: - assert call['method'] == 'POST' - assert call['uri'] == url + await provider._upload_part(file_stream, provider.CHUNK_SIZE, upload_args) + + assert aiohttpretty.has_call(method='POST', uri=upload_part_url) + + provider.CHUNK_SIZE = CHUNK_SIZE @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_complete_session(self, provider, stream_10_MB, provider_fixtures): + async def test_complete_session(self, provider, file_stream, provider_fixtures): - url = provider._build_content_url('files', 'upload_session', 'finish') + assert file_stream.size == 38 + provider.CHUNK_SIZE = 4 + path = WaterButlerPath('/foobah') + session_id = provider_fixtures.get('session_metadata').get('session_id', '') + + complete_part_url = provider._build_content_url('files', 'upload_session', 'finish') aiohttpretty.register_json_uri( 'POST', - url, + complete_part_url, status=200, - body=provider_fixtures['file_metadata'] + body=provider_fixtures.get('file_metadata', None) ) + metadata = await provider._complete_session(file_stream, session_id, path) - upload_data = await provider._complete_session( - stream_10_MB, - 'session id', - {'path': 'hedgehogs'} - ) + assert metadata == provider_fixtures.get('file_metadata', None) + assert aiohttpretty.has_call(method='POST', uri=complete_part_url) - assert upload_data == provider_fixtures['file_metadata'] - assert aiohttpretty.has_call(method='POST', uri=url) + provider.CHUNK_SIZE = CHUNK_SIZE @pytest.mark.asyncio @pytest.mark.aiohttpretty diff --git a/tests/utils.py b/tests/utils.py index 475260ddc..42b467c75 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,9 +7,7 @@ from unittest import mock import pytest -import aiohttp -from tornado import testing -from tornado.concurrent import Future +from tornado import concurrent, testing from tornado.platform.asyncio import AsyncIOMainLoop from waterbutler.server.app import make_app @@ -18,19 +16,6 @@ from waterbutler.core.streams.file import FileStreamReader -class MockStream(aiohttp.streams.StreamReader): - def __init__(self, data): - super().__init__() - if isinstance(data, str): - data = data.encode('UTF-8') - elif not isinstance(data, bytes): - raise TypeError('Data must be either str or bytes, found {!r}'.format(type(data))) - - self.size = len(data) - self.feed_data(data) - self.feed_eof() - - class MockCoroutine(mock.Mock): if sys.version_info >= (3, 5, 3): @@ -90,7 +75,7 @@ def __init__(self): super().__init__(tempfile.TemporaryFile()) -class MockRequestBody(Future): +class MockRequestBody(concurrent.Future): def __await__(self): yield None