From 1408a9bdbd86908a250575942ac35b68a2a49d7a Mon Sep 17 00:00:00 2001 From: seibed Date: Tue, 25 Sep 2018 16:18:03 -0400 Subject: [PATCH] Completed version 1 of folder download feature. Code works properly now. --- .../ecs/browser/spring/ServiceController.java | 98 ++++++++++++++++++- .../resources/static/javascript/S3Browser.js | 20 +++- .../static/javascript/S3BrowserUtil.js | 28 ++++++ .../static/javascript/S3TemplateEngine.js | 2 + .../resources/static/javascript/ecs-sdk.js | 34 ++++++- 5 files changed, 177 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/emc/ecs/browser/spring/ServiceController.java b/src/main/java/com/emc/ecs/browser/spring/ServiceController.java index af2afe3..96962d9 100644 --- a/src/main/java/com/emc/ecs/browser/spring/ServiceController.java +++ b/src/main/java/com/emc/ecs/browser/spring/ServiceController.java @@ -15,8 +15,16 @@ package com.emc.ecs.browser.spring; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; @@ -32,6 +40,7 @@ import org.eclipse.jetty.util.StringUtil; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RequestMapping; @@ -52,7 +61,9 @@ import com.emc.object.s3.bean.ListObjectsResult; import com.emc.object.s3.bean.ListVersionsResult; import com.emc.object.s3.bean.QueryObjectsResult; +import com.emc.object.s3.bean.S3Object; import com.emc.object.s3.bean.VersioningConfiguration; +import com.emc.object.s3.jersey.S3JerseyClient; import com.emc.object.s3.request.PresignedUrlRequest; import com.emc.object.s3.bean.SlimCopyObjectResult; import com.emc.object.util.RestUtil; @@ -164,9 +175,49 @@ public ResponseEntity postProxy(HttpServletRequest request) throws Exception presignedUrlRequest.setNamespace(s3Config.getNamespace()); S3SignerV2 s3Signer = new S3SignerV2(s3Config); dataToReturn = s3Signer.generatePresignedUrl(presignedUrlRequest).toString(); - } else { + } else if ("download".equals(request.getHeader("X-Passthrough-Type"))) { + String downloadFolder = request.getHeader("X-Passthrough-Download-Folder"); + if (StringUtil.isBlank(downloadFolder)) { + downloadFolder = "/usr/src/app/"; + } + sign(method.toString(), resource, parameters, headers, s3Config); + + HttpHeaders newHeaders = new HttpHeaders(); + for (Entry> header : headers.entrySet()) { + List headerValue = new ArrayList(header.getValue().size()); + for (Object value : header.getValue()) { + headerValue.add((String) value); + } + newHeaders.put(header.getKey(), headerValue); + } + + String queryString = RestUtil.generateRawQueryString(parameters); + if (StringUtil.isNotBlank(queryString)) { + resource = resource + "?" + queryString; + } + String endpoint = request.getHeader("X-Passthrough-Endpoint"); + while (endpoint.endsWith("/")) { + endpoint = endpoint.substring(0, endpoint.length() - 1); + } + resource = endpoint + resource; + RequestEntity requestEntity = new RequestEntity(data, newHeaders, method, new URI(resource)); + RestTemplate client = new RestTemplate(); + try { + dataToReturn = client.exchange(requestEntity, ListObjectsResult.class); + if ( ((ResponseEntity) dataToReturn).getStatusCode().is2xxSuccessful() ) { + ListObjectsResult objectsToDownload = ((ResponseEntity) dataToReturn).getBody(); + downloadObjects( downloadFolder, objectsToDownload.getBucketName(), objectsToDownload.getObjects(), s3Config ); + } + } catch (HttpClientErrorException e) { + dataToReturn = new ErrorData(e); // handle and display on the other end + } catch (Exception e) { + dataToReturn = new ErrorData(e); // handle and display on the other end + } + } else { + sign(method.toString(), resource, parameters, headers, s3Config); + HttpHeaders newHeaders = new HttpHeaders(); for (Entry> header : headers.entrySet()) { List headerValue = new ArrayList(header.getValue().size()); @@ -225,6 +276,39 @@ public ResponseEntity postProxy(HttpServletRequest request) throws Exception return ResponseEntity.ok( dataToReturn ); } + /** + * @param downloadFolder + * @param objects + * @param s3Config + * @throws Exception + */ + private void downloadObjects(String downloadFolder, String bucketName, List objects, S3Config s3Config) throws Exception { + File downloadBucketParent = new File( downloadFolder ); + File downloadBucket = new File( downloadBucketParent, bucketName ); + if ( !downloadBucket.exists() ) { + if ( !downloadBucket.mkdirs() ) { + throw new Exception( "Download location cannot be created: " + downloadBucket.getAbsolutePath() ); + } + } else if ( downloadBucket.isDirectory() ) { + throw new Exception( "Download location is not a folder: " + downloadBucket.getAbsolutePath() ); + } + + S3JerseyClient client = new S3JerseyClient( s3Config ); + for ( S3Object object : objects ) { + String key = object.getKey(); + File file = new File( downloadBucket, key ); + if ( !file.getParentFile().exists() ) { + file.getParentFile().mkdirs(); + } + final Path destination = Paths.get(file.getAbsolutePath()); + try ( + final InputStream inputStream = client.getObject(bucketName, key).getObject(); + ) { + Files.copy(inputStream, destination); + } + } + } + /** * @param request * @return @@ -236,7 +320,7 @@ private S3Config getS3Config(HttpServletRequest request) throws Exception { String passthroughAccessKey = request.getHeader("X-Passthrough-Key"); String passthroughSecretKey = request.getHeader("X-Passthrough-Secret"); while (passthroughEndpoint.endsWith("/")) { - passthroughEndpoint = passthroughEndpoint.substring(0, passthroughEndpoint.length() - 1); + passthroughEndpoint = passthroughEndpoint.substring(0, passthroughEndpoint.length() - 1); } S3Config s3Config = new S3Config(new URI(passthroughEndpoint)); s3Config.setIdentity(passthroughAccessKey); @@ -368,6 +452,16 @@ public ErrorData(HttpClientErrorException e) { responseBody = e.getResponseBodyAsString(); } + /** + * @param e + */ + public ErrorData(Exception e) { + status = 500; + statusText = "Server Error"; + message = e.getMessage(); + responseBody = ""; + } + public int getStatus() { return status; } diff --git a/src/main/resources/static/javascript/S3Browser.js b/src/main/resources/static/javascript/S3Browser.js index 6220d0d..cddc2aa 100644 --- a/src/main/resources/static/javascript/S3Browser.js +++ b/src/main/resources/static/javascript/S3Browser.js @@ -334,18 +334,22 @@ S3Browser.prototype.list = function( entry, extraQueryParameters ) { browser.filterRows(); }, extraQueryParameters ); }; + S3Browser.prototype.resetCurrentEntry = function( locationText ) { this.currentEntry = this.util.getCurrentEntry( locationText ); this.refresh(); }; + S3Browser.prototype.refresh = function() { this.list( this.currentEntry ); }; + S3Browser.prototype.openFile = function( entry ) { this.util.getShareableUrl( entry, this.util.futureDate( 1, 'hours' ), function( data ) { window.open( data ); } ); }; + S3Browser.prototype.openSelectedItems = function() { console.trace(); var selectedRows = this.getSelectedRows(); @@ -364,9 +368,19 @@ S3Browser.prototype.downloadSelectedItems = function() { console.log( this.util.getLocationText( this.currentEntry ) ); var selectedRows = this.getSelectedRows(); if ( selectedRows.length == 0 ) this.util.error( this.templates.get( 'nothingSelectedError' ).render() ); - if ( !this._checkNoDirectories( selectedRows ) ) return; for ( i = 0; i < selectedRows.length; i++ ) { - this.util.downloadFile( selectedRows[i].entry ); + if ( selectedRows[i].entry.type == FileRow.ENTRY_TYPE.REGULAR ) { + this.util.downloadFile( selectedRows[i].entry ); + } else { + var browser = this; + + var downloadFolder = browser.util.prompt('downloadFolderPrompt', {}, browser.util.validDownloadFolder, 'validDownloadFolderError', '/usr/src/app'); + if ( ( downloadFolder == null ) || ( downloadFolder.length == 0 ) ) { + return; + } + + this.util.downloadFolder( selectedRows[i].entry, downloadFolder ); + } } }; @@ -376,12 +390,14 @@ S3Browser.prototype.showProperties = function( entry ) { new PropertiesPage( entry, browser.util, browser.templates ); } ); }; + S3Browser.prototype.showAcl = function( entry ) { var browser = this; this.util.getAcl( entry, function( acl ) { new AclPage( entry, acl, browser.util, browser.templates ); } ); }; + S3Browser.prototype.showObjectInfo = function( entry ) { if ( this.util.isListable( entry.type ) ) { this.util.error( this.templates.get( 'directoryNotAllowedError' ).render() ); diff --git a/src/main/resources/static/javascript/S3BrowserUtil.js b/src/main/resources/static/javascript/S3BrowserUtil.js index 051f318..3874b0f 100644 --- a/src/main/resources/static/javascript/S3BrowserUtil.js +++ b/src/main/resources/static/javascript/S3BrowserUtil.js @@ -355,6 +355,10 @@ S3BrowserUtil.prototype.validName = function(name) { return !(!name || name.trim().length == 0 || /[/]/.test(name)); }; +S3BrowserUtil.prototype.validDownloadFolder = function(downloadFolder) { + return true; +}; + S3BrowserUtil.prototype.endWithDelimiter = function(path) { path = path.trim(); if (path.charAt(path.length - 1) !== _s3Delimiter) @@ -831,6 +835,30 @@ S3BrowserUtil.prototype.downloadFile = function( entry ) { }); }; +S3BrowserUtil.prototype.downloadFolder = function( entry, downloadFolder ) { + console.trace(); + var util = this; + var parameters = { + entry: entry, + downloadFolder: downloadFolder + }; + if ( this.useHierarchicalMode ) { + parameters.delimiter = _s3Delimiter; + } + var folderToDownload = util.getLocationText( entry ); + var successMessage = 'Successfully downloaded ' + folderToDownload; + var failureMessage = 'Failure while downloading ' + folderToDownload; + this.showStatus('Downloading folder...'); + this.s3.downloadFolder( parameters, function( error, data ) { + util.hideStatus('Downloading folder...'); + if ( error == null ) { + alert( successMessage ); + } else { + alert( failureMessage ); + } + }); +}; + S3BrowserUtil.prototype.ifExists = function( entry, existsCallback, notExistsCallback, errorCallback ) { var util = this; this.showStatus('Checking existence...'); diff --git a/src/main/resources/static/javascript/S3TemplateEngine.js b/src/main/resources/static/javascript/S3TemplateEngine.js index 2cbe0bf..cf16966 100644 --- a/src/main/resources/static/javascript/S3TemplateEngine.js +++ b/src/main/resources/static/javascript/S3TemplateEngine.js @@ -108,6 +108,8 @@ S3TemplateEngine.MESSAGE_TEMPLATES = { configDataCorruptPrompt: 'Your configuration data has been corrupted and will be reset.', deleteUidPrompt: 'Are you sure you want to delete the following UID?\n%{token.uid}', storageDisabledPrompt: 'Browser data storage seems to be disabled\n(are you in private browsing mode?)\nYour credentials cannot be saved,\nbut will be available until the browser window is closed.', + downloadFolderPrompt: 'What is the folder where your downloaded content should be stored?', + validDownloadFolderError: '%{downloadFolder} does not exist, and cannot be used as a download folder.', bucketCors:'CORS not configured for bucket %{bucketName}', configPageTitle: 'Configuration', uidPageTitle: 'Add UID', diff --git a/src/main/resources/static/javascript/ecs-sdk.js b/src/main/resources/static/javascript/ecs-sdk.js index e611ead..b0108d0 100644 --- a/src/main/resources/static/javascript/ecs-sdk.js +++ b/src/main/resources/static/javascript/ecs-sdk.js @@ -40,7 +40,8 @@ function combineWithDelimiter( part1, part2 ) { }; function handleData( data, callback, dataProcessor ) { - if ( data.status && ( ( data.status >= 200 ) && ( data.status < 300 ) ) ) { + if ( ( data.status && ( ( data.status >= 200 ) && ( data.status < 300 ) ) ) + || ( data.statusCode && data.statusCode == 'OK' ) ) { if (dataProcessor) { data = dataProcessor( data ); } @@ -421,6 +422,37 @@ EcsS3.prototype.restoreVersion = function( entry, callback ) { }); }; +EcsS3.prototype.downloadFolder = function( params, callback ) { + var apiUrl = this.getBucketApiUrl( params.entry ); + var separatorChar = '?'; + if ( isNonEmptyString( params.entry.key ) ) { + apiUrl = apiUrl + separatorChar + 'prefix=' + params.entry.key; + separatorChar = '&'; + } + if ( isNonEmptyString( params.delimiter ) ) { + apiUrl = apiUrl + separatorChar + 'delimiter=' + params.delimiter; + separatorChar = '&'; + } + if ( isNonEmptyString( params.extraQueryParameters ) ) { + apiUrl = apiUrl + separatorChar + params.extraQueryParameters; + separatorChar = '&'; + } + var headers = this.getHeaders('GET'); + headers['X-Passthrough-Type'] = 'download'; + if ( isNonEmptyString( params.downloadFolder ) ) { + headers['X-Passthrough-Download-Folder'] = params.downloadFolder; + } + + $.ajax({ url: apiUrl, method: 'POST', headers: headers, + success: function(data, textStatus, jqHXR) { + handleData( data, callback ); + }, + error: function(jqHXR, textStatus, errorThrown) { + handleError( callback, jqHXR, errorThrown, textStatus ); + }, + }); +}; + EcsS3.prototype.getServiceInformation = function( callback ) { var apiUrl = this.getSystemApiUrl() + '?endpoint'; var headers = this.getHeaders('GET');