Fix issues with s3 storage support

cropduster/files.py

@@ -4,8 +4,7 @@
 import re
 import hashlib
 
-import PIL.Image
-
+from django.core.files.images import get_image_dimensions
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.files.storage import default_storage
 from django.conf import settings
@@ -45,11 +44,11 @@ def delete(self, *args, **kwargs):
     @cached_property
     def dimensions(self):
         try:
-            pil_image = PIL.Image.open(self.path)
+            close = self.closed
+            self.open()
+            return get_image_dimensions(self, close=close)
         except:
             return (0, 0)
-        else:
-            return pil_image.size
 
     @cached_property
     def width(self):

cropduster/forms.py

@@ -83,9 +83,15 @@ def get_option_attrs(self, value):
             except (TypeError, self.model.DoesNotExist):
                 return {}
 
+        if thumb.image_id:
+            thumb_url = thumb.image.get_image_url(size_name=thumb.name)
+        else:
+            thumb_url = None
+
         return {
             'data-width': thumb.width,
             'data-height': thumb.height,
+            'data-url': thumb_url,
             'data-tmp-file': json.dumps(not(thumb.image_id)),
         }
 

cropduster/static/cropduster/js/cropduster.js

@@ -114,7 +114,12 @@ window.CropDuster = {};
 
     var image_css = function(src, width, height, opts, is_ie) {
         var css = '';
-        src = encodeURI(src || '') + '?v=' + randomDigits(9);
+        src = src || '';
+        if (src.indexOf('%') === -1) {
+          src = encodeURI(src);
+          var q = src.indexOf('?') > -1 ? '&' : '?';
+          src = encodeURI(src || '') + q + 'v=' + randomDigits(9);
+        }
         css += 'background-image:url("' + src + '");';
 
         if (width > 750) {
@@ -219,6 +224,7 @@ window.CropDuster = {};
                     'value': thumb.id,
                     'data-width': thumb.width,
                     'data-height': thumb.height,
+                    'data-url': thumb.url,
                     'data-tmp-file': 'true',
                     'selected': 'selected'
                 });
@@ -351,7 +357,7 @@ window.CropDuster = {};
                 // double slashes that don't follow a colon.
                 url = url.replace(/(:)?\/+/g, function($0, $1) { return $1 ? $0 : '/'; });
                 thumbData[slug] = {
-                    'image_url': url,
+                    'image_url': data.url,
                     'size_slug': slug,
                     'width': data.width,
                     'height': data.height

cropduster/tests/helpers.py

@@ -4,11 +4,15 @@
 import shutil
 import uuid
 
-from django.conf import settings
+import PIL.Image
+
 from django.core.files.storage import default_storage
 from django.core.files.base import ContentFile
 from django.test import override_settings
 
+from .utils import repr_rgb
+
+
 PATH = os.path.split(__file__)[0]
 ORIG_IMG_PATH = os.path.join(PATH, 'data')
 
@@ -17,26 +21,15 @@ class CropdusterTestCaseMediaMixin(object):
 
     def _pre_setup(self):
         super(CropdusterTestCaseMediaMixin, self)._pre_setup()
-        new_settings = {}
         self.temp_media_root = tempfile.mkdtemp(prefix='TEST_MEDIA_ROOT_')
-
-        storage_cls = settings.DEFAULT_FILE_STORAGE
-
-        if storage_cls == 'django.core.files.storage.FileSystemStorage':
-            new_settings['MEDIA_ROOT'] = self.temp_media_root
-        elif storage_cls != 'storages.backends.s3boto3.S3Boto3Storage':
-            raise Exception("Unsupported DEFAULT_FILE_STORAGE %s" % storage_cls)
-        self.override = override_settings(**new_settings)
+        self.override = override_settings(MEDIA_ROOT=self.temp_media_root)
         self.override.enable()
 
     def _post_teardown(self):
         if hasattr(default_storage, 'bucket'):
             default_storage.bucket.objects.filter(Prefix=default_storage.location).delete()
-        else:
-            shutil.rmtree(self.temp_media_root)
-
+        shutil.rmtree(self.temp_media_root)
         self.override.disable()
-
         super(CropdusterTestCaseMediaMixin, self)._post_teardown()
 
     def setUp(self):
@@ -46,6 +39,26 @@ def setUp(self):
         self.TEST_IMG_DIR = ORIG_IMG_PATH
         self.TEST_IMG_DIR_RELATIVE = os.path.join(random, 'data')
 
+    def assertImageColorEqual(self, element, image):
+        self.selenium.execute_script('arguments[0].scrollIntoView()', element)
+        scroll_top = -1 * self.selenium.execute_script(
+            'return document.body.getBoundingClientRect().top')
+        tmp_file = tempfile.NamedTemporaryFile(suffix='.png')
+        if not self.selenium.save_screenshot(tmp_file.name):
+            raise Exception("Failed to save screenshot")
+        pixel_density = self.selenium.execute_script('return window.devicePixelRatio') or 1
+        im1 = PIL.Image.open(tmp_file.name).convert('RGB')
+        x1 = int(round(element.location['x'] + (element.size['width'] // 2.0)))
+        y1 = int(round(element.location['y'] - scroll_top + (element.size['height'] // 2.0)))
+        rgb1 = im1.getpixel((x1 * pixel_density, y1 * pixel_density))
+        im2 = PIL.Image.open(os.path.join(os.path.dirname(__file__), 'data', image)).convert('RGB')
+        w, h = im2.size
+        x2, y2 = int(round(w // 2.0)), int(round(h // 2.0))
+        rgb2 = im2.getpixel((x2, y2))
+        if rgb1 != rgb2:
+            msg = "Colors differ: %s != %s" % (repr_rgb(rgb1), repr_rgb(rgb2))
+            self.fail(msg)
+
     def create_unique_image(self, image):
         image_uuid = uuid.uuid4().hex
 

cropduster/tests/settings.py

@@ -26,7 +26,8 @@
 if os.environ.get('S3') == '1':
     DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
     AWS_STORAGE_BUCKET_NAME = 'ollie-cropduster-media-test-bucket-dev'
-    AWS_DEFAULT_ACL = None
+    AWS_DEFAULT_ACL = 'public-read'
     AWS_LOCATION = 'cropduster/%s/' % uuid.uuid4().hex
+    AWS_S3_SIGNATURE_VERSION = 's3v4'
 else:
     DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'

cropduster/tests/test_admin.py

@@ -2,6 +2,7 @@
 
 import os
 
+from django.core.files.storage import default_storage
 from selenosis import AdminSelenosisTestCase
 
 from cropduster.models import Image, Size
@@ -42,7 +43,7 @@ def test_addform_single_image(self):
 
         with self.switch_to_popup_window():
             with self.visible_selector('#id_image') as el:
-                el.send_keys(os.path.join(self.TEST_IMG_DIR, 'img.jpg'))
+                el.send_keys(os.path.join(self.TEST_IMG_DIR, 'img.png'))
             with self.clickable_selector('#upload-button') as el:
                 el.click()
             with self.clickable_selector('#crop-button') as el:
@@ -52,7 +53,7 @@ def test_addform_single_image(self):
 
         author = Author.objects.all()[0]
         sizes = list(Size.flatten(Author.HEADSHOT_SIZES))
-        self.assertTrue(bool(author.headshot.path))
+        self.assertTrue(bool(author.headshot.name))
 
         image = author.headshot.related_object
         thumbs = image.thumbs.all()
@@ -83,7 +84,7 @@ def test_addform_single_image(self):
             'image_id': image.pk,
             'id': auto_thumb.pk,
         })
-        self.assertTrue(os.path.exists(auto_thumb.path))
+        self.assertTrue(default_storage.exists(auto_thumb.image_name))
 
     def test_addform_multiple_image(self):
         author = Author.objects.create(name="Mark Twain")
@@ -129,13 +130,13 @@ def test_addform_multiple_image(self):
         lead_sizes = list(Size.flatten(Article.LEAD_IMAGE_SIZES))
         alt_sizes = list(Size.flatten(Article.ALT_IMAGE_SIZES))
 
-        self.assertTrue(article.lead_image.path.endswith('.jpg'))
+        self.assertTrue(article.lead_image.name.endswith('.jpg'))
         self.assertEqual(len(article.lead_image.related_object.thumbs.all()), len(lead_sizes))
-        self.assertTrue(article.alt_image.path.endswith('.png'))
+        self.assertTrue(article.alt_image.name.endswith('.png'))
         self.assertEqual(len(article.alt_image.related_object.thumbs.all()), len(alt_sizes))
 
     def test_changeform_single_image(self):
-        image_path = self.create_unique_image('img.jpg')
+        image_path = self.create_unique_image('img.png')
         author = Author.objects.create(name="Samuel Langhorne Clemens",
             headshot=image_path)
         Image.objects.create(image=image_path, content_object=author)
@@ -144,6 +145,10 @@ def test_changeform_single_image(self):
 
         self.load_admin(author)
 
+        preview_image_el = self.selenium.find_element_by_css_selector('#headshot-group .cropduster-image span')
+        src_image_path = os.path.join(self.TEST_IMG_DIR, 'img.png')
+        self.assertImageColorEqual(preview_image_el, src_image_path)
+
         elem = self.selenium.find_element_by_id('id_name')
         elem.clear()
         elem.send_keys("Mark Twain")

cropduster/tests/test_utils.py

@@ -1,8 +1,12 @@
+from io import open, BytesIO
 import os
 import shutil
+import tempfile
+
 from PIL import Image
 
 from django import test
+from django.core.files.storage import default_storage
 from django.conf import settings
 
 from .helpers import CropdusterTestCaseMediaMixin
@@ -18,20 +22,24 @@ def test_that_test_work(self):
 
     def test_get_image_extension(self):
         from ..utils import get_image_extension
-        shutil.copyfile(
-            os.path.join(self.TEST_IMG_DIR, 'img.jpg'),
-            os.path.join(self.TEST_IMG_DIR, 'jpg_bad_ext.pdf'))
 
-        shutil.copyfile(
-            os.path.join(self.TEST_IMG_DIR, 'img.png'),
-            os.path.join(self.TEST_IMG_DIR, 'png_bad_ext.jpg'))
+        tmp_jpg_bad_ext_pdf = tempfile.NamedTemporaryFile(suffix='.pdf')
+        tmp_png_bad_ext_jpg = tempfile.NamedTemporaryFile(suffix='.png')
+
+        with open(os.path.join(self.TEST_IMG_DIR, 'img.jpg'), mode='rb') as f:
+            tmp_jpg_bad_ext_pdf.write(f.read())
+            tmp_jpg_bad_ext_pdf.seek(0)
+
+        with open(os.path.join(self.TEST_IMG_DIR, 'img.png'), mode='rb') as f:
+            tmp_png_bad_ext_jpg.write(f.read())
+            tmp_png_bad_ext_jpg.seek(0)
 
         imgs = [
             (self._get_img('img.jpg'), '.jpg'),
             (self._get_img('img.png'), '.png'),
             (self._get_img('animated.gif'), '.gif'),
-            (self._get_img('jpg_bad_ext.pdf'), '.jpg'),
-            (self._get_img('png_bad_ext.jpg'), '.png'),
+            (Image.open(tmp_jpg_bad_ext_pdf.name), '.jpg'),
+            (Image.open(tmp_png_bad_ext_jpg.name), '.png'),
         ]
         for img, ext in imgs:
             self.assertEqual(get_image_extension(img), ext)
@@ -68,11 +76,10 @@ def test_get_upload_foldername(self):
 
         path = random = uuid.uuid4().hex
         folder_path = get_upload_foldername('my img.jpg', upload_to=path)
-        self.assertEqual(folder_path, os.path.join(path, 'my_img'))
-        os.mkdir(os.path.join(settings.MEDIA_ROOT, folder_path))
+        self.assertEqual(folder_path, "%s/my_img" % (path))
+        default_storage.save("%s/original.jpg" % folder_path, BytesIO(b''))
         self.assertEqual(get_upload_foldername('my img.jpg', upload_to=path),
                          os.path.join(path, 'my_img-1'))
-        shutil.rmtree(os.path.join(settings.MEDIA_ROOT, path))
 
     def test_get_min_size(self):
         from ..utils import get_min_size

cropduster/tests/test_views.py

@@ -1,15 +1,14 @@
 import os
 
 from django import test
+from django.core.files.storage import default_storage
 try:
     from django.urls import reverse
 except ImportError:
     from django.core.urlresolvers import reverse
 from django.contrib.auth.models import User
 from django.http import HttpRequest
 
-from generic_plus.utils import get_media_path
-
 from cropduster import views
 from cropduster.utils import json
 
@@ -109,8 +108,6 @@ def test_post_request(self):
         request = self.factory.post(reverse('cropduster-upload'), data)
         request.user = self.user
         response = views.upload(request)
-        data = json.loads(response.content)
-        uploaded_img_path = get_media_path(data['url'])
-
         self.assertEqual(response.status_code, 200)
-        self.assertTrue(os.path.exists(uploaded_img_path))
+        data = json.loads(response.content)
+        self.assertTrue(default_storage.exists(data['orig_image']))

cropduster/tests/utils.py

@@ -0,0 +1,40 @@
+import six
+
+
+def esc_code(codes=None):
+    if codes is None:
+        # reset escape code
+        return "\x1b[0m"
+    if not isinstance(codes, (list, tuple)):
+        codes = [codes]
+    return '\x1b[0;' + ';'.join(map(six.text_type, codes)) + 'm'
+
+
+def get_luminance(rgb):
+    rgb_map = []
+    for val in rgb:
+        val = val / 256
+        if val <= 0.03928:
+            rgb_map.append(val / 12.92)
+        else:
+            rgb_map.append(pow((val + 0.055) / 1.055, 2.4))
+
+    return (0.2126 * rgb_map[0]) + (0.7152 * rgb_map[1]) + (0.0722 * rgb_map[2])
+
+
+def repr_rgb(rgb):
+    r, g, b = rgb
+    codes = (48, 2, r, g, b)
+    reset = "\x1b[0m"
+    hex_color = "#%s" % ("".join(["%02x" % c for c in rgb]))
+    luminance = get_luminance(rgb)
+    if luminance > 0.5:
+        codes += (38, 2, 0, 0, 0)
+    else:
+        codes += (38, 2, 255, 255, 255)
+
+    return "%(codes)s%(hex)s%(reset)s" % {
+        'codes': esc_code(codes),
+        'hex': hex_color,
+        'reset': reset,
+    }

cropduster/views/__init__.py

@@ -425,6 +425,7 @@ def crop(request):
 
             for name, new_thumb in six.iteritems(new_thumbs):
                 thumb_data = dict([(k, getattr(new_thumb, k)) for k in json_thumb_fields])
+                thumb_data['url'] = db_image.get_image_url(name, tmp=not(new_thumb.image_id))
                 crop_data['thumbs'].update({name: thumb_data})
                 if new_thumb.reference_thumb_id:
                     continue