diff --git a/.gitignore b/.gitignore index 230e51cf..94ac30f9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ hs_err_pid* /bin/ /target/ +*.classpath +*.project diff --git a/pom.xml b/pom.xml index 5510e872..6708137e 100644 --- a/pom.xml +++ b/pom.xml @@ -280,4 +280,4 @@ - + \ No newline at end of file diff --git a/src/main/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlan.java b/src/main/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlan.java index 13b5b261..1956412f 100644 --- a/src/main/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlan.java +++ b/src/main/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlan.java @@ -53,6 +53,12 @@ public class BundleDownloadPlan { * List of files not included in the bundle */ private NotIncludedFile[] notIncluded; + + private long size; + + private long bundleCount; + + private long filesCount; /** * Default Constructor @@ -77,13 +83,17 @@ public BundleDownloadPlan() { * If files are not included in the bundle. */ public BundleDownloadPlan(String postEachTo, String status, BundleRequest[] requests, - String[] messages, NotIncludedFile[] notIncluded) + String[] messages, NotIncludedFile[] notIncluded, long size, long bundleCount, long filesCount) { this.postEachTo = postEachTo; this.status = status; this.requests = requests; this.messages = messages; this.notIncluded = notIncluded; + this.size = size; + this.bundleCount = bundleCount; + this.filesCount = filesCount; + } /** @@ -132,7 +142,31 @@ public void setMessages(String[] msgs) { public void setNotIncluded(NotIncludedFile[] notIncluded) { this.notIncluded = notIncluded; } + + public void setSize(long size) { + this.size = size; + } + + public long getSize() { + return this.size; + } + + + public void setBundleCount(long bCount) { + this.bundleCount = bCount; + } + public long getBundleCount() { + return this.bundleCount; + } + + public void setFilesCount(long fCount) { + this.filesCount = fCount; + } + + public long getFilesCount() { + return this.filesCount; + } /** * return the API endpoint to POST each request in the plan to. * @@ -183,5 +217,6 @@ public String[] getMessages() { public NotIncludedFile[] getNotIncluded() { return this.notIncluded; } + } diff --git a/src/main/java/gov/nist/oar/distrib/datapackage/BundleRequest.java b/src/main/java/gov/nist/oar/distrib/datapackage/BundleRequest.java index 613b8c78..eca5e71a 100644 --- a/src/main/java/gov/nist/oar/distrib/datapackage/BundleRequest.java +++ b/src/main/java/gov/nist/oar/distrib/datapackage/BundleRequest.java @@ -28,6 +28,8 @@ public class BundleRequest { * FilePAths and Urls json array */ private FileRequest[] includeFiles; + + private long bundleSize; /** * Default Constructor @@ -42,11 +44,30 @@ public BundleRequest() { * @param bundleName * @param includeFiles */ - public BundleRequest(String bundleName, FileRequest[] includeFiles) { + public BundleRequest(String bundleName, FileRequest[] includeFiles, long bundleSize) { this.bundleName = bundleName; this.includeFiles = includeFiles; + this.bundleSize = bundleSize; } + /** + * Set the bundle size + * + * @param bundleName the name to give to the bundle + */ + public void setBundleSize(long size) { + this.bundleSize = size; + } + + /** + * Return the name requested to be assigned to the bundle + * + * @return String -- the desired name for the bundle + */ + public long getBundleSize() { + return this.bundleSize; + } + /** * Set the requested name to give to the bundle * diff --git a/src/main/java/gov/nist/oar/distrib/datapackage/DefaultDataPackager.java b/src/main/java/gov/nist/oar/distrib/datapackage/DefaultDataPackager.java index 5bb3ee6d..2be515ed 100644 --- a/src/main/java/gov/nist/oar/distrib/datapackage/DefaultDataPackager.java +++ b/src/main/java/gov/nist/oar/distrib/datapackage/DefaultDataPackager.java @@ -48,110 +48,382 @@ */ public class DefaultDataPackager implements DataPackager { - private long mxFileSize; - private int mxFilesCount; - private FileRequest[] inputfileList; - private BundleRequest bundleRequest; - private String domains; - private int allowedRedirects; - - private int fileCount; - private StringBuilder bundlelogfile = new StringBuilder(""); - private StringBuilder bundlelogError = new StringBuilder(""); - private List listUrlsStatusSize = new ArrayList<>(); - protected static Logger logger = LoggerFactory.getLogger(DefaultDataPackager.class); - private long totalRequestedPackageSize = -1; - private int requestValidity = 0; - private ValidationHelper validationHelper = new ValidationHelper(); - - public DefaultDataPackager() { - // Default Constructor - } - /** - * Construct input parameters to be used within the class - * - * @param inputjson - * requested Bundle - * @param maxFileSize - * total file size allowed to download - * @param numOfFiles - * total number of files allowed to download - */ - public DefaultDataPackager(BundleRequest inputjson, long maxFileSize, int numOfFiles, String domains, int allowedRedirects) { - this.bundleRequest = inputjson; - this.mxFileSize = maxFileSize; - this.mxFilesCount = numOfFiles; - this.domains = domains; - this.allowedRedirects = allowedRedirects; - } + private long mxFileSize; + private int mxFilesCount; + private FileRequest[] inputfileList; + private BundleRequest bundleRequest; + private String domains; + private int allowedRedirects; + + private int fileCount; + private StringBuilder bundlelogfile = new StringBuilder(""); + private StringBuilder bundlelogError = new StringBuilder(""); + private List listUrlsStatusSize = new ArrayList<>(); + protected static Logger logger = LoggerFactory.getLogger(DefaultDataPackager.class); + private long totalRequestedPackageSize = -1; + private int requestValidity = 0; + - /*** - * Read inputstream from valid urls and stream it to outputstream provided - * by response handler. - * - * @param zout - * ZipOutputStream - * @throws DistributionException - * @throws IOException - */ + public DefaultDataPackager() { + // Default Constructor + } - @Override - public void getData(ZipOutputStream zout) throws IOException, DistributionException { - HttpURLConnection con = null; - this.validateBundleRequest(); - - logger.debug("Forming zip file from the the input fileurls"); - - for (int i = 0; i < inputfileList.length; i++) { - FileRequest jobject = inputfileList[i]; - String filepath = jobject.getFilePath(); - String downloadurl = jobject.getDownloadUrl(); - if(this.validateUrl(downloadurl)) { - URLStatusLocation uLoc = listUrlsStatusSize.get(i); - if ((downloadurl.equalsIgnoreCase(uLoc.getRequestedURL())) && this.checkResponse(uLoc)) { - InputStream fstream = null; - try { - URL obj = new URL(uLoc.getRequestedURL()); - con = (HttpURLConnection) obj.openConnection(); - fstream = con.getInputStream(); - int len; - byte[] buf = new byte[100000]; - zout.putNextEntry(new ZipEntry(filepath)); - while ((len = fstream.read(buf)) != -1) { - zout.write(buf, 0, len); - } - zout.closeEntry(); - fstream.close(); - fileCount++; - } catch (IOException ie) { - bundlelogError.append("\n Exception in getting data for: " + filepath + " at " + - downloadurl+ ";\n this file might be corrupted."); - logger.error("There is an error reading this file from: " + downloadurl + ": " - + ie.getMessage() + "\n"+formatLocation(ie, "getData")); - - // clean up - if (con != null) - quietClose(con.getErrorStream(), uLoc.getRequestedURL()+" (error stream)"); - zout.closeEntry(); - } finally { - quietClose(fstream, uLoc.getRequestedURL()); - if (con != null) - con.disconnect(); - } - } - } + /** + * Construct input parameters to be used within the class + * + * @param inputjson requested Bundle + * @param maxFileSize total file size allowed to download + * @param numOfFiles total number of files allowed to download + */ + public DefaultDataPackager(BundleRequest inputjson, long maxFileSize, int numOfFiles, String domains, + int allowedRedirects) { + this.bundleRequest = inputjson; + this.mxFileSize = maxFileSize; + this.mxFilesCount = numOfFiles; + this.domains = domains; + this.allowedRedirects = allowedRedirects; } - if (fileCount == 0) { - logger.warn("The package does not contain any data. These errors :" + this.bundlelogError); - throw new NoContentInPackageException("No data or files written in Bundle/Package."); + /*** + * Read inputstream from valid urls and stream it to outputstream provided by + * response handler. + * + * @param zout ZipOutputStream + * @throws DistributionException + * @throws IOException + */ + + @Override + public void getData(ZipOutputStream zout) throws IOException, DistributionException { + HttpURLConnection con = null; + this.validateBundleRequest(); + + logger.info("Forming zip file from the the input fileurls"); + + for (int i = 0; i < inputfileList.length; i++) { + FileRequest jobject = inputfileList[i]; + String filepath = jobject.getFilePath(); + String downloadurl = jobject.getDownloadUrl(); + if (this.validateUrl(downloadurl)) { + URLStatusLocation uLoc = listUrlsStatusSize.get(i); + if ((downloadurl.equalsIgnoreCase(uLoc.getRequestedURL())) && this.checkResponse(uLoc)) { + InputStream fstream = null; + try { + URL obj = new URL(uLoc.getRequestedURL()); + con = (HttpURLConnection) obj.openConnection(); + fstream = con.getInputStream(); + int len; + byte[] buf = new byte[100000]; + zout.putNextEntry(new ZipEntry(filepath)); + while ((len = fstream.read(buf)) != -1) { + zout.write(buf, 0, len); + } + zout.closeEntry(); + fstream.close(); + fileCount++; + } catch (IOException ie) { + bundlelogError.append("\n Exception in getting data for: " + filepath + " at " + downloadurl + + "\n" + "This file might be corrupt."); + logger.error("There is an error reading this file at location: " + downloadurl + "Exception: " + + ie.getMessage()); + zout.closeEntry(); + } finally { + if (fstream != null) + fstream.close(); + if (con != null) + con.disconnect(); + } + + } + } + } + + if (fileCount == 0) { + logger.info("The package does not contain any data. These errors :" + this.bundlelogError); + throw new NoContentInPackageException("No data or files written in Bundle/Package."); + } + this.writeLog(zout); + } - this.writeLog(zout); - } + /** + * This method/function accepts URLStatusLocation to check the status of the URL + * response. Creates appropriate log messages and returns true on successfully + * accessing data. + * + * @param URLStatusLocation + * @return boolean + * @throws IOException + */ + private boolean checkResponse(URLStatusLocation uloc) { + + if (uloc.getStatus() == 200) { + return true; + } else if (uloc.getStatus() >= 300) { + dealWithErrors(uloc); + return false; + } else if (uloc.getStatus() == 0) { + this.bundlelogError.append("\n " + uloc.getRequestedURL()); + if (!uloc.isValidURL()) { + this.bundlelogError.append( + " does not belong to allowed/valid domains, so this file is not downnloaded in the bundle/package."); + } else { + + this.bundlelogError.append(" There is an Error accessing this file. Could not connect successfully. "); + } + return false; + } + return false; + } + + /** + * If the error code is returned, handle the error and create proper response + * message , write in log string builder. + * + * @param uloc + * @throws IOException + */ + private void dealWithErrors(URLStatusLocation uloc) { + + String requestedUrl = uloc.getRequestedURL(); + if (uloc.getStatus() >= 400 && uloc.getStatus() <= 500) { - /* + logger.info(requestedUrl + " Error accessing this url: " + uloc.getStatus()); + this.bundlelogError.append("\n " + requestedUrl); + this.bundlelogError + .append(" There is an Error accessing this file, Server returned status with response code "); + this.bundlelogError + .append(uloc.getStatus() + " and message:" + ValidationHelper.getStatusMessage(uloc.getStatus())); + + } else if (uloc.getStatus() >= 300 && uloc.getStatus() <= 400) { + + this.bundlelogError.append("\n" + requestedUrl + " There are too many redirects for this URL."); + + } else if (uloc.getStatus() >= 500) { + this.bundlelogError.append("\n" + requestedUrl + " There is an internal server error accessing this url."); + this.bundlelogError.append(" Server returned status with response code :" + uloc.getStatus()); + + } + + } + + /** + * Write a log file in the bundle/package based on the response from accessing + * requested URLs + * + * @param zout + * @throws IOException + */ + private void writeLog(ZipOutputStream zout) throws IOException { + InputStream nStream = null; + try { + String filename = ""; + int l; + byte[] buf = new byte[10000]; + StringBuilder bundleInfo = new StringBuilder( + "Information about requested bundle/package is given below.\n"); + if (bundlelogfile.length() != 0) { + bundleInfo.append("\n Following files are not included in the bundle for the reasons given: \n"); + bundleInfo.append(this.bundlelogfile); + filename = "/PackagingErrors.txt"; + } + + if (bundlelogError.length() != 0) { + bundleInfo.append( + "\n Following files are not included in the bundle because of errors: \n" + bundlelogError); + filename = "/PackagingErrors.txt"; + } + + if ((bundlelogfile.length() == 0 && bundlelogError.length() == 0) && !listUrlsStatusSize.isEmpty()) { + bundleInfo.append("\n All requested files are successfully added to this bundle."); + filename = "/PackagingSuccessful.txt"; + } + + nStream = new ByteArrayInputStream(bundleInfo.toString().getBytes()); + zout.putNextEntry(new ZipEntry(filename)); + while ((l = nStream.read(buf)) != -1) { + zout.write(buf, 0, l); + } + zout.closeEntry(); + } catch (IOException ie) { + logger.info("Exception while creating Ziplogfile" + ie.getMessage()); + if (zout != null) + zout.closeEntry(); + } finally { + if (nStream != null) + nStream.close(); + + } + } + + + /** + * If called for the first time within the Datapackager life cycle, this + * function/method validates the request. It checks and removes duplicates, + * checks total file size to compare with allowed size, Checks total number of + * files allowed. This is to validate request and validate input JSON data sent + * by the client. If this function is already called by a DataPackager, code + * uses existing non-zero status to return appropriate error message, to avoid + * redoing HEAD calls to get required information for validation. + */ + @Override + public void validateBundleRequest() throws IOException, DistributionException { + if (requestValidity == 0) { + + try { + basicValidation(); + + if (this.inputfileList.length <= 0) { + requestValidity = 3; + + } else { + ValidationHelper.removeDuplicates(this.inputfileList); + long totalFilesSize = this.getTotalSize(); + + if (totalFilesSize > this.mxFileSize && this.getFilesCount() > 1) { + requestValidity = 4; + + } else if (this.getFilesCount() > this.mxFilesCount) { + requestValidity = 5; + } else { + int countNotAccessible = ValidationHelper.noOfNotAcceccibleURLs(this.listUrlsStatusSize); + if (countNotAccessible == this.getFilesCount()) { + requestValidity = 6; + + } else if (requestValidity == 0) + requestValidity = 1; + } + } + + } catch (IOException ie) { + requestValidity = 2; + } + + } + + getServiceErrorStatus(requestValidity); + + } + + /** + * This method, helps check validation status internally to avoid doing it + * multiple times. As validation status is cached for the DataPackager life + * cycle this helps return appropriate exception if any without redoing HEAD/Get + * calls. + * + * @param status takes requestValidity variable input + * @throws IOException + * @throws DistributionException + */ + private void getServiceErrorStatus(int status) throws IOException, DistributionException { + switch (status) { + case 1: + requestValidity = 1; + return; + case 2: + throw new IOException("IOException while validating request and parsing input."); + case 3: + throw new EmptyBundleRequestException(); + case 4: + throw new InputLimitException("Total filesize is beyond allowed limit of " + this.mxFileSize); + case 5: + throw new InputLimitException( + "Total number of files requested is beyond allowed limit " + this.mxFilesCount); + case 6: + throw new NoFilesAccesibleInPackageException("None of the URLs returned data requested."); + default: + return; + } + } + + /** + * Function to get total files size requested by client throws IOException if + * there is an issue accessing url. + * + * @return long, total size. + * @throws MalformedURLException + * @throws IOException + */ + + public long getTotalSize() throws IOException { + + if (this.totalRequestedPackageSize == -1) { + basicValidation(); + List list = Arrays.asList(this.inputfileList); + + List downloadurls = list.stream().map(FileRequest::getDownloadUrl).collect(Collectors.toList()); + long totalSize = 0; + + for (int i = 0; i < downloadurls.size(); i++) { + URLStatusLocation uLoc = ValidationHelper.getFileURLStatusSize(downloadurls.get(i), this.domains, + this.allowedRedirects); + listUrlsStatusSize.add(uLoc); + totalSize += uLoc.getLength(); + } + this.totalRequestedPackageSize = totalSize; + } + return totalRequestedPackageSize; + } + + /** + * Checks whether total number of files requested are within allowed limit. + * + * @return boolean based on comparison + * @throws IOException + */ + + private int getFilesCount() throws IOException { + basicValidation(); + return this.inputfileList.length; + } + + /** + * Validate requested URL by checking whether it is from allowed domains. + */ + @Override + public boolean validateUrl(String url) { + try { + if (!ValidationHelper.isAllowedURL(url, this.domains)) { + this.bundlelogfile.append("\n Url here:" + url); + this.bundlelogfile.append(" does not belong to allowed domains, so this file is " + + "not downnloaded in the bundle/package."); + return false; + } + return true; + } catch (MalformedURLException ex) { + this.bundlelogfile.append("\n Url here:" + url); + this.bundlelogfile + .append(", is not a legal URL, so this file is " + "not downnloaded in the bundle/package."); + return false; + } + } + + /** + * Read the name from Bundle Request, if no name is provided, default name + * prefix 'download' is returned + */ + @Override + public String getBundleName() throws IOException { + String bundleName; + basicValidation(); + return ((bundleName = bundleRequest.getBundleName()) != null) ? bundleName : "download"; + + } + + /** + * This checks whether file list is populated if not parse input JSON, validate + * and get files. + * + * @throws IOException + */ + private void basicValidation() throws IOException { + if (this.inputfileList == null) { + JSONUtils.isJSONValid(this.bundleRequest); + this.inputfileList = this.bundleRequest.getIncludeFiles(); + } + } + /* * provide a compact printing of the stack trace that focuses on the frame(s) associated * with a particular class and method */ @@ -195,283 +467,5 @@ private void quietClose(Closeable c, String name) { } } - /** - * This method/function accepts URLStatusLocation to check the status of the - * URL response. Creates appropriate log messages and returns true on - * successfully accessing data. - * - * @param URLStatusLocation - * @return boolean - * @throws IOException - */ - private boolean checkResponse(URLStatusLocation uloc) { - - if (uloc.getStatus() == 200) { - return true; - } else if (uloc.getStatus() >= 300) { - dealWithErrors(uloc); - return false; - } else if (uloc.getStatus() == 0) { - this.bundlelogError.append("\n " + uloc.getRequestedURL()); - if (!uloc.isValidURL()) { - this.bundlelogError.append( - " does not belong to allowed/valid domains, so this file is not downnloaded in the bundle/package."); - } else { - - this.bundlelogError.append(" There is an Error accessing this file. Could not connect successfully. "); - } - return false; - } - return false; - } - - /** - * If the error code is returned, handle the error and create proper - * response message , write in log string builder. - * - * @param uloc - * @throws IOException - */ - private void dealWithErrors(URLStatusLocation uloc) { - - String requestedUrl = uloc.getRequestedURL(); - if (uloc.getStatus() >= 400 && uloc.getStatus() <= 500) { - - logger.error(requestedUrl + " Error accessing this url: " + uloc.getStatus()); - this.bundlelogError.append("\n " + requestedUrl); - this.bundlelogError - .append(" There is an Error accessing this file, Server returned status with response code "); - this.bundlelogError - .append(uloc.getStatus() + " and message:" + ValidationHelper.getStatusMessage(uloc.getStatus())); - - } else if (uloc.getStatus() >= 300 && uloc.getStatus() <= 400) { - - this.bundlelogError.append("\n" + requestedUrl + " There are too many redirects for this URL."); - - } else if (uloc.getStatus() >= 500) { - this.bundlelogError.append("\n" + requestedUrl + " There is an internal server error accessing this url."); - this.bundlelogError.append(" Server returned status with response code :" + uloc.getStatus()); - - } - - } - - /** - * Write a log file in the bundle/package based on the response from - * accessing requested URLs - * - * @param zout - * @throws IOException - */ - private void writeLog(ZipOutputStream zout) throws IOException { - InputStream nStream = null; - try { - String filename = ""; - int l; - byte[] buf = new byte[10000]; - StringBuilder bundleInfo = new StringBuilder( - "Information about requested bundle/package is given below.\n"); - if (bundlelogfile.length() != 0) { - bundleInfo.append("\n Following files are not included in the bundle for the reasons given: \n"); - bundleInfo.append(this.bundlelogfile); - filename = "/PackagingErrors.txt"; - } - - if (bundlelogError.length() != 0) { - bundleInfo.append( - "\n Following files are not included in the bundle because of errors: \n" + bundlelogError); - filename = "/PackagingErrors.txt"; - } - - if ((bundlelogfile.length() == 0 && bundlelogError.length() == 0) && !listUrlsStatusSize.isEmpty()) { - bundleInfo.append("\n All requested files are successfully added to this bundle."); - filename = "/PackagingSuccessful.txt"; - } - - nStream = new ByteArrayInputStream(bundleInfo.toString().getBytes()); - zout.putNextEntry(new ZipEntry(filename)); - while ((l = nStream.read(buf)) != -1) { - zout.write(buf, 0, l); - } - zout.closeEntry(); - } catch (IOException ie) { - logger.warn("Exception while creating Ziplogfile" + ie.getMessage()); - if(zout != null) - zout.closeEntry(); - } - finally { - if(nStream != null) - nStream.close(); - } - } - - /** - * If called for the first time within the Datapackager life cycle, this - * function/method validates the request. It checks and removes duplicates, - * checks total file size to compare with allowed size, Checks total number - * of files allowed. This is to validate request and validate input JSON - * data sent by the client. If this function is already called by a - * DataPackager, code uses existing non-zero status to return appropriate - * error message, to avoid redoing HEAD calls to get required information - * for validation. - */ - @Override - public void validateBundleRequest() throws IOException, DistributionException { - if (requestValidity == 0) { - - try { - basicValidation(); - - if (this.inputfileList.length <= 0) { - requestValidity = 3; - - } else { - ValidationHelper.removeDuplicates(this.inputfileList); - long totalFilesSize = this.getTotalSize(); - - if (totalFilesSize > this.mxFileSize && this.getFilesCount() > 1) { - requestValidity = 4; - - } else if (this.getFilesCount() > this.mxFilesCount) { - requestValidity = 5; - } else { - int countNotAccessible = ValidationHelper.noOfNotAcceccibleURLs(this.listUrlsStatusSize); - if (countNotAccessible == this.getFilesCount()) { - requestValidity = 6; - - } - else if (requestValidity == 0) - requestValidity = 1; - } - } - - } catch (IOException ie) { - requestValidity = 2; - } - - } - - getServiceErrorStatus(requestValidity); - - } - - /** - * This method, helps check validation status internally to avoid doing it - * multiple times. As validation status is cached for the DataPackager life - * cycle this helps return appropriate exception if any without redoing - * HEAD/Get calls. - * - * @param status - * takes requestValidity variable input - * @throws IOException - * @throws DistributionException - */ - private void getServiceErrorStatus(int status) throws IOException, DistributionException { - switch (status) { - case 1: - requestValidity = 1; - return; - case 2: - throw new IOException("IOException while validating request and parsing input."); - case 3: - throw new EmptyBundleRequestException(); - case 4: - throw new InputLimitException("Total filesize is beyond allowed limit of " + this.mxFileSize); - case 5: - throw new InputLimitException( - "Total number of files requested is beyond allowed limit " + this.mxFilesCount); - case 6: - throw new NoFilesAccesibleInPackageException("None of the URLs returned data requested."); - default: - return; - } - } - - /** - * Function to get total files size requested by client throws IOException - * if there is an issue accessing url. - * - * @return long, total size. - * @throws MalformedURLException - * @throws IOException - */ - - public long getTotalSize() throws IOException { - - if (this.totalRequestedPackageSize == -1) { - basicValidation(); - List list = Arrays.asList(this.inputfileList); - - List downloadurls = list.stream().map(FileRequest::getDownloadUrl).collect(Collectors.toList()); - long totalSize = 0; - - for (int i = 0; i < downloadurls.size(); i++) { - URLStatusLocation uLoc = ValidationHelper.getFileURLStatusSize(downloadurls.get(i), this.domains, this.allowedRedirects); - listUrlsStatusSize.add(uLoc); - totalSize += uLoc.getLength(); - } - this.totalRequestedPackageSize = totalSize; - } - return totalRequestedPackageSize; - } - - /** - * Checks whether total number of files requested are within allowed limit. - * - * @return boolean based on comparison - * @throws IOException - */ - - private int getFilesCount() throws IOException { - basicValidation(); - return this.inputfileList.length; - } - - /** - * Validate requested URL by checking whether it is from allowed domains. - */ - @Override - public boolean validateUrl(String url) { - try { - if (!ValidationHelper.isAllowedURL(url, this.domains)) { - this.bundlelogfile.append("\n Url here:" + url); - this.bundlelogfile.append(" does not belong to allowed domains, so this file is "+ - "not downnloaded in the bundle/package."); - return false; - } - return true; - } - catch (MalformedURLException ex) { - this.bundlelogfile.append("\n Url here:" + url); - this.bundlelogfile.append(", is not a legal URL, so this file is "+ - "not downnloaded in the bundle/package."); - return false; - } - } - - /** - * Read the name from Bundle Request, if no name is provided, default name - * prefix 'download' is returned - */ - @Override - public String getBundleName() throws IOException { - String bundleName; - basicValidation(); - return ((bundleName = bundleRequest.getBundleName()) != null) ? bundleName : "download"; - - } - - /** - * This checks whether file list is populated if not parse input JSON, - * validate and get files. - * - * @throws IOException - */ - private void basicValidation() throws IOException { - if (this.inputfileList == null) { - JSONUtils.isJSONValid(this.bundleRequest); - this.inputfileList = this.bundleRequest.getIncludeFiles(); - } - } } diff --git a/src/main/java/gov/nist/oar/distrib/datapackage/DownloadBundlePlanner.java b/src/main/java/gov/nist/oar/distrib/datapackage/DownloadBundlePlanner.java index 3a5b2bb1..16f405e5 100644 --- a/src/main/java/gov/nist/oar/distrib/datapackage/DownloadBundlePlanner.java +++ b/src/main/java/gov/nist/oar/distrib/datapackage/DownloadBundlePlanner.java @@ -41,186 +41,190 @@ */ public class DownloadBundlePlanner { - protected static Logger logger = LoggerFactory.getLogger(DownloadBundlePlanner.class); - List notIncludedFiles; - List filePathUrls; - List bundleFilePathUrls; - List messages; - BundleDownloadPlan finalPlan; - long bundleSize = 0; - int bundledFilesCount = 0; - int bundleCount = 0; - String bundleName = ""; - boolean unsupportedSource; - String status = "complete"; - private FileRequest[] inputfileList; - private BundleRequest bundleRequest; - private long mxFilesBundleSize; - private int mxBundledFilesCount; - private String validdomains; - private int allowedRedirects; - private ValidationHelper validationHelper = new ValidationHelper(); - - public DownloadBundlePlanner() { - // Default constructor - } - - public DownloadBundlePlanner(BundleRequest inputjson, long maxFileSize, int numOfFiles, String validdomains, - String bundleName, int allowedRedirects) { - this.bundleRequest = inputjson; - this.mxFilesBundleSize = maxFileSize; - this.mxBundledFilesCount = numOfFiles; - this.validdomains = validdomains; - this.bundleName = bundleName; - this.allowedRedirects = allowedRedirects; - } - - /** - * Get the plan to download all files after checking various limits and - * criteria - * - * @return BundleDownloadPlan -- the recommended plan - * @throws DistributionException - */ - public BundleDownloadPlan getBundleDownloadPlan() throws DistributionException { - - notIncludedFiles = new ArrayList(); - filePathUrls = new ArrayList(); - bundleFilePathUrls = new ArrayList(); - messages = new ArrayList(); - logger.info("Creating bundle plan.."); - try { - ObjectMapper mapper = new ObjectMapper(); - String requestString = mapper.writeValueAsString(this.bundleRequest); - if (ValidationHelper.hasHTMLTags(requestString)) { - messages.add("Input contains html code, make sure to post proper request."); - this.status = "Error"; - return makeBundlePlan(); - - } - } catch (JsonProcessingException ex) { - // should not happen - logger.error("There is an issue validating request. unable to create valid JSON."); - throw new DistributionException("Trouble validating request: unable to convert to JSON: " + - ex.getMessage()); - } - - try { - JSONUtils.isJSONValid(this.bundleRequest); - this.inputfileList = this.bundleRequest.getIncludeFiles(); - ValidationHelper.removeDuplicates(this.inputfileList); - } catch (IOException ie) { - logger.error("Error while parsing request, not a valid JSON input."+ie.getMessage()); - messages.add("Error while parsing the request. Check if it is valid JSON."); - this.status = "Error"; - return makeBundlePlan(); - + protected static Logger logger = LoggerFactory.getLogger(DownloadBundlePlanner.class); + List notIncludedFiles; + List filePathUrls; + List bundleFilePathUrls; + List messages; + BundleDownloadPlan finalPlan; + long bundleSize = 0; + int bundledFilesCount = 0; + int bundleCount = 0; + String bundleName = ""; + boolean unsupportedSource; + String status = "complete"; + private FileRequest[] inputfileList; + private BundleRequest bundleRequest; + private long mxFilesBundleSize; + private int mxBundledFilesCount; + private String validdomains; + private int allowedRedirects; + private long totalRequestedFileSize = 0; + + public DownloadBundlePlanner() { + // Default constructor } - for (int i = 0; i < inputfileList.length; i++) { - FileRequest jobject = inputfileList[i]; - String filepath = jobject.getFilePath(); - String downloadurl = jobject.getDownloadUrl(); - try { - if (ValidationHelper.isAllowedURL(downloadurl, validdomains)) { - this.makeBundles(jobject); - } else { - notIncludedFiles.add(new NotIncludedFile(filepath, downloadurl, - "File not added in package; This URL is from unsupported domain/host.")); - } - } catch (MalformedURLException ex) { - notIncludedFiles.add(new NotIncludedFile(filepath, downloadurl, - "File not added in package; malformed URL")); - } + public DownloadBundlePlanner(BundleRequest inputjson, long maxFileSize, int numOfFiles, String validdomains, + String bundleName, int allowedRedirects) { + this.bundleRequest = inputjson; + this.mxFilesBundleSize = maxFileSize; + this.mxBundledFilesCount = numOfFiles; + this.validdomains = validdomains; + this.bundleName = bundleName; + this.allowedRedirects = allowedRedirects; } - if (!this.filePathUrls.isEmpty()) { - this.makePlan(this.filePathUrls); - } + /** + * Get the plan to download all files after checking various limits and criteria + * + * @return BundleDownloadPlan -- the recommended plan + * @throws DistributionException + */ + public BundleDownloadPlan getBundleDownloadPlan() throws DistributionException { + + notIncludedFiles = new ArrayList(); + filePathUrls = new ArrayList(); + bundleFilePathUrls = new ArrayList(); + messages = new ArrayList(); + logger.info("Creating bundle plan.."); + try { + ObjectMapper mapper = new ObjectMapper(); + String requestString = mapper.writeValueAsString(this.bundleRequest); + if (ValidationHelper.hasHTMLTags(requestString)) { + messages.add("Input contains html code, make sure to post proper request."); + this.status = "Error"; + this.bundleCount = 0; + return makeBundlePlan(); + + } + } catch (JsonProcessingException ex) { + // should not happen + logger.error("There is an issue validating request. unable to create valid JSON."); + throw new DistributionException( + "Trouble validating request: unable to convert to JSON: " + ex.getMessage()); + } + + try { + JSONUtils.isJSONValid(this.bundleRequest); + this.inputfileList = this.bundleRequest.getIncludeFiles(); + ValidationHelper.removeDuplicates(this.inputfileList); + } catch (IOException ie) { + logger.error("Error while parsing request, not a valid JSON input." + ie.getMessage()); + messages.add("Error while parsing the request. Check if it is valid JSON."); + this.status = "Error"; + return makeBundlePlan(); + + } + + for (int i = 0; i < inputfileList.length; i++) { + FileRequest jobject = inputfileList[i]; + String filepath = jobject.getFilePath(); + String downloadurl = jobject.getDownloadUrl(); + try { + if (ValidationHelper.isAllowedURL(downloadurl, validdomains)) { + this.makeBundles(jobject); + } else { + notIncludedFiles.add(new NotIncludedFile(filepath, downloadurl, + "File not added in package; This URL is from unsupported domain/host.")); + } + } catch (MalformedURLException ex) { + notIncludedFiles + .add(new NotIncludedFile(filepath, downloadurl, "File not added in package; malformed URL")); + } + } + + if (!this.filePathUrls.isEmpty()) { + this.makePlan(this.filePathUrls); + } + + this.updateMessagesAndStatus(); + return this.makeBundlePlan(); - this.updateMessagesAndStatus(); - return this.makeBundlePlan(); - - } - - /** - * Add to Bundle of the input requested based on the size and number of - * files allowed per bundle request. - * - * @param jobject - * @throws IOException - */ - public void makeBundles(FileRequest jobject) { - logger.info("Make bundles planning."); - bundledFilesCount++; - URLStatusLocation uObj = ValidationHelper.getFileURLStatusSize(jobject.getDownloadUrl(), this.validdomains, this.allowedRedirects); - long individualFileSize = uObj.getLength(); - String whyNotIncluded = "File not added in package; "; - if(uObj.getStatus() >=300 && uObj.getStatus() <400) { - whyNotIncluded += "There are too many redirects for this URL."; - notIncludedFiles.add(new NotIncludedFile(jobject.getFilePath(), jobject.getDownloadUrl(), - whyNotIncluded)); } - else if (individualFileSize <= 0) { + + /** + * Add to Bundle of the input requested based on the size and number of files + * allowed per bundle request. + * + * @param jobject + * @throws IOException + */ + public void makeBundles(FileRequest jobject) { + logger.info("Make bundles: validate urls, check size and accordinlgy create bundle plan."); + bundledFilesCount++; + URLStatusLocation uObj = ValidationHelper.getFileURLStatusSize(jobject.getDownloadUrl(), this.validdomains, + this.allowedRedirects); + long individualFileSize = uObj.getLength(); + String whyNotIncluded = "File not added in package; "; + if (uObj.getStatus() >= 300 && uObj.getStatus() < 400) { + whyNotIncluded += "There are too many redirects for this URL."; + notIncludedFiles.add(new NotIncludedFile(jobject.getFilePath(), jobject.getDownloadUrl(), whyNotIncluded)); + } else if (individualFileSize <= 0) { // String whyNotIncluded = "File not added in package; "; // if(uObj.getStatus() >=300 && uObj.getStatus() <400) // whyNotIncluded += "There are too many redirects for this URL."; // else - whyNotIncluded += ValidationHelper.getStatusMessage(uObj.getStatus()); - notIncludedFiles.add(new NotIncludedFile(jobject.getFilePath(), jobject.getDownloadUrl(), - whyNotIncluded)); - } else { - if (individualFileSize >= this.mxFilesBundleSize) { - List onefilePathUrls = new ArrayList(); - onefilePathUrls.add(new FileRequest(jobject.getFilePath(), jobject.getDownloadUrl())); - this.makePlan(onefilePathUrls); - } else { - bundleSize += individualFileSize; - if (bundleSize < this.mxFilesBundleSize && bundledFilesCount <= this.mxBundledFilesCount) { - filePathUrls.add(new FileRequest(jobject.getFilePath(), jobject.getDownloadUrl())); + whyNotIncluded += ValidationHelper.getStatusMessage(uObj.getStatus()); + notIncludedFiles.add(new NotIncludedFile(jobject.getFilePath(), jobject.getDownloadUrl(), whyNotIncluded)); } else { - makePlan(filePathUrls); - filePathUrls.clear(); - bundledFilesCount = 1; - filePathUrls.add(new FileRequest(jobject.getFilePath(), jobject.getDownloadUrl())); - bundleSize = individualFileSize; + totalRequestedFileSize += individualFileSize; + if (individualFileSize >= this.mxFilesBundleSize) { + List onefilePathUrls = new ArrayList(); + onefilePathUrls.add(new FileRequest(jobject.getFilePath(), jobject.getDownloadUrl())); + this.makePlan(onefilePathUrls); + } else { + bundleSize += individualFileSize; + if (bundleSize < this.mxFilesBundleSize && bundledFilesCount <= this.mxBundledFilesCount) { + filePathUrls.add(new FileRequest(jobject.getFilePath(), jobject.getDownloadUrl())); + } else { + makePlan(filePathUrls); + filePathUrls.clear(); + bundledFilesCount = 1; + filePathUrls.add(new FileRequest(jobject.getFilePath(), jobject.getDownloadUrl())); + bundleSize = individualFileSize; + } + } } - } } - } - - /*** - * Create Bundle of FileList - * - * @param fPathUrls - */ - public void makePlan(List fPathUrls) { - bundleCount++; - FileRequest[] bundlefilePathUrls = fPathUrls.toArray(new FileRequest[0]); - bundleFilePathUrls.add(new BundleRequest(bundleName + "-" + bundleCount + ".zip", bundlefilePathUrls)); - } - - private void updateMessagesAndStatus() { - if (!this.notIncludedFiles.isEmpty() && this.bundleFilePathUrls.isEmpty()) { - this.messages.add("No Files added in the Bundle, there are problems accessing URLs."); - this.status = "Error"; + + /*** + * Create Bundle of FileList + * + * @param fPathUrls + */ + public void makePlan(List fPathUrls) { + bundleCount++; + FileRequest[] bundlefilePathUrls = fPathUrls.toArray(new FileRequest[0]); + bundleFilePathUrls.add(new BundleRequest(bundleName + "-" + bundleCount + ".zip", bundlefilePathUrls, bundleSize)); } - if (!this.notIncludedFiles.isEmpty() && !this.bundleFilePathUrls.isEmpty()) { - messages.add("Some URLs have problem accessing contents."); - this.status = "warnings"; + + /** + * Update Messages and Status + */ + private void updateMessagesAndStatus() { + if (!this.notIncludedFiles.isEmpty() && this.bundleFilePathUrls.isEmpty()) { + this.messages.add("No Files added in the Bundle, there are problems accessing URLs."); + this.status = "Error"; + } + if (!this.notIncludedFiles.isEmpty() && !this.bundleFilePathUrls.isEmpty()) { + messages.add("Some URLs have problem accessing contents."); + this.status = "warnings"; + } + } - } - - /** - * Create final Bundle plan (JSON) to return to client. It creates Java - * Object of BundleDownloadPlan after processing input request. - */ - public BundleDownloadPlan makeBundlePlan() { - logger.info("makeBundlePlan called."); - return new BundleDownloadPlan("_bundle", this.status, - bundleFilePathUrls.toArray(new BundleRequest[0]), messages.toArray(new String[0]), - notIncludedFiles.toArray(new NotIncludedFile[0])); - } + /** + * Create final Bundle plan (JSON) to return to client. It creates Java Object + * of BundleDownloadPlan after processing input request. + */ + public BundleDownloadPlan makeBundlePlan() { + logger.info("makeBundlePlan called to return bundleDownloadPlan with urls and sizes."); + int fileList = 0 ; + if(inputfileList != null ) fileList = inputfileList.length; + return new BundleDownloadPlan("_bundle", this.status, bundleFilePathUrls.toArray(new BundleRequest[0]), + messages.toArray(new String[0]), notIncludedFiles.toArray(new NotIncludedFile[0]), + this.totalRequestedFileSize, bundleCount, fileList); + } } diff --git a/src/main/resources/bootstrap.yml b/src/main/resources/bootstrap.yml index c9002467..2558bde3 100644 --- a/src/main/resources/bootstrap.yml +++ b/src/main/resources/bootstrap.yml @@ -39,4 +39,4 @@ security: logging: file: distservice.log path : /var/log/dist-service - exception-conversion-word: '%wEx' + exception-conversion-word: '%wEx' \ No newline at end of file diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index e8a4a369..2b1fce6e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -44,4 +44,4 @@ - + \ No newline at end of file diff --git a/src/test/java/gov/nist/oar/distrib/DownloadBundlePlannerTest.java b/src/test/java/gov/nist/oar/distrib/DownloadBundlePlannerTest.java index 6fb54595..d0050021 100644 --- a/src/test/java/gov/nist/oar/distrib/DownloadBundlePlannerTest.java +++ b/src/test/java/gov/nist/oar/distrib/DownloadBundlePlannerTest.java @@ -53,7 +53,7 @@ public void getBundleDownloadPlanTest() FileRequest testval2 = mapper.readValue(val2, FileRequest.class); inputfileList[0] = testval1; inputfileList[1] = testval2; - BundleRequest bFL = new BundleRequest("testdownload", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload", inputfileList, 0); DownloadBundlePlanner dpl = new DownloadBundlePlanner(bFL, 200000, 3, "s3.amazonaws.com|project-open-data.cio.gov", "testdownload", 1); BundleDownloadPlan bundlePlan = dpl.getBundleDownloadPlan(); @@ -80,7 +80,7 @@ public void getBundleDownloadPlan2Test() inputfileList[0] = testval1; inputfileList[1] = testval2; inputfileList[2] = testval3; - BundleRequest bFL = new BundleRequest("testdownload", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload", inputfileList,0); DownloadBundlePlanner dpl = new DownloadBundlePlanner(bFL, 2000000, 3, "s3.amazonaws.com|nist.gov", "testdownload", 1); BundleDownloadPlan bundlePlan = dpl.getBundleDownloadPlan(); @@ -108,7 +108,7 @@ public void getBundleDownloadPlan3Test() inputfileList[0] = testval1; inputfileList[1] = testval2; inputfileList[2] = testval3; - BundleRequest bFL = new BundleRequest("testdownload", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload", inputfileList,0); DownloadBundlePlanner dpl = new DownloadBundlePlanner(bFL, 200, 2, "s3.amazonaws.com|nist.gov", "testdownload", 1); BundleDownloadPlan bundlePlan = dpl.getBundleDownloadPlan(); @@ -138,7 +138,7 @@ public void getBundleDownloadPlan4Test() ifileList[0] = fileRequest1; ifileList[1] = fileRequest2; - BundleRequest bFL = new BundleRequest("testdownload", ifileList); + BundleRequest bFL = new BundleRequest("testdownload", ifileList,0); DownloadBundlePlanner dpl = new DownloadBundlePlanner(bFL, 200, 2, "s3.amazonaws.com|nist.gov|httpstat.us", "testdownload",7); BundleDownloadPlan bundlePlan = dpl.getBundleDownloadPlan(); diff --git a/src/test/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlanTest.java b/src/test/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlanTest.java index 7684e123..ca22445b 100644 --- a/src/test/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlanTest.java +++ b/src/test/java/gov/nist/oar/distrib/datapackage/BundleDownloadPlanTest.java @@ -28,14 +28,14 @@ public BundleRequest[] makeBundles() { FileRequest fpathUrl_2 = new FileRequest("/filepath/file-2.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); BundleRequest bundle1 = new BundleRequest("download_data_1", - new FileRequest[] { fpathUrl_1, fpathUrl_2 }); + new FileRequest[] { fpathUrl_1, fpathUrl_2 },0); FileRequest fpathUrl_3 = new FileRequest("/filepath-2/testfile-1.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); FileRequest fpathUrl_4 = new FileRequest("/filepath-2/testfile-2.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); BundleRequest bundle2 = new BundleRequest("download_data_2", - new FileRequest[] { fpathUrl_3, fpathUrl_4 }); + new FileRequest[] { fpathUrl_3, fpathUrl_4 },0); return new BundleRequest[] { bundle1, bundle2 }; } @@ -56,7 +56,7 @@ public NotIncludedFile[] makeNotIncluded() { @Test public void testBundleDownloadPlan() { BundleDownloadPlan bundlePlan = new BundleDownloadPlan("_bundle", "partial", this.makeBundles(), - this.makeMessages(), this.makeNotIncluded()); + this.makeMessages(), this.makeNotIncluded(),0,0,0); assertEquals("_bundle", bundlePlan.getPostEachTo()); assertEquals("partial", bundlePlan.getStatus()); diff --git a/src/test/java/gov/nist/oar/distrib/datapackage/BundleNameFilePathUrlTest.java b/src/test/java/gov/nist/oar/distrib/datapackage/BundleNameFilePathUrlTest.java index 28d44f64..95316e9d 100644 --- a/src/test/java/gov/nist/oar/distrib/datapackage/BundleNameFilePathUrlTest.java +++ b/src/test/java/gov/nist/oar/distrib/datapackage/BundleNameFilePathUrlTest.java @@ -34,7 +34,7 @@ public void testBundleNameFilePathUrl() { FileRequest fpathUrl_2 = new FileRequest("/filepath/file-2.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); BundleRequest bundle1 = new BundleRequest("download_data_1", - new FileRequest[] { fpathUrl_1, fpathUrl_2 }); + new FileRequest[] { fpathUrl_1, fpathUrl_2 },0); assertEquals("download_data_1", bundle1.getBundleName()); assertEquals("/filepath/file-1.pdf", bundle1.getIncludeFiles()[0].getFilePath()); @@ -49,14 +49,15 @@ public void testJson() throws JsonProcessingException, JSONException { String fileUrl2 = "{\"filePath\":\"/filepath/file-2.pdf\",\"downloadUrl\":\"https://s3.amazonaws.com/nist-midas/1894/license.pdf\"}"; String bundleNAme = "\"bundleName\":\"download_data_1\" "; - String bundleJson = "{" + bundleNAme + "," + "\"includeFiles\"" + ":[" + fileUrl1 + "," + fileUrl2 + "]" + "}"; + String bundleJson = "{" + bundleNAme + "," + "\"includeFiles\"" + ":[" + fileUrl1 + "," + fileUrl2 + "]" + + ", \"bundleSize\": 66 }"; FileRequest fpathUrl_1 = new FileRequest("/filepath/file-1.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); FileRequest fpathUrl_2 = new FileRequest("/filepath/file-2.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); BundleRequest bundle1 = new BundleRequest("download_data_1", - new FileRequest[] { fpathUrl_1, fpathUrl_2 }); + new FileRequest[] { fpathUrl_1, fpathUrl_2 },66); String json = new ObjectMapper().writeValueAsString(bundle1); JSONAssert.assertEquals(bundleJson, json, true); @@ -68,14 +69,14 @@ public BundleRequest[] makeBundles() { FileRequest fpathUrl_2 = new FileRequest("/filepath/file-2.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); BundleRequest bundle1 = new BundleRequest("download_data_1", - new FileRequest[] { fpathUrl_1, fpathUrl_2 }); + new FileRequest[] { fpathUrl_1, fpathUrl_2 },0); FileRequest fpathUrl_3 = new FileRequest("/filepath-2/testfile-1.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); FileRequest fpathUrl_4 = new FileRequest("/filepath-2/testfile-2.pdf", "https://s3.amazonaws.com/nist-midas/1894/license.pdf"); BundleRequest bundle2 = new BundleRequest("download_data_2", - new FileRequest[] { fpathUrl_3, fpathUrl_4 }); + new FileRequest[] { fpathUrl_3, fpathUrl_4 },0); return new BundleRequest[] { bundle1, bundle2 }; } diff --git a/src/test/java/gov/nist/oar/distrib/datapackage/DefaultDataPackagerTest.java b/src/test/java/gov/nist/oar/distrib/datapackage/DefaultDataPackagerTest.java index ea89ab1a..d793feef 100644 --- a/src/test/java/gov/nist/oar/distrib/datapackage/DefaultDataPackagerTest.java +++ b/src/test/java/gov/nist/oar/distrib/datapackage/DefaultDataPackagerTest.java @@ -176,7 +176,7 @@ private static void createBundleRequest() throws IOException { FileRequest testval2 = mapper.readValue(val2, FileRequest.class); inputfileList[0] = testval1; inputfileList[1] = testval2; - bundleRequest = new BundleRequest("testdatabundle", inputfileList); + bundleRequest = new BundleRequest("testdatabundle", inputfileList,0); dp = new DefaultDataPackager(bundleRequest, mxFileSize, numberofFiles, domains, redirectURLTrials); } diff --git a/src/test/java/gov/nist/oar/distrib/service/DefaultDataPackagingServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/DefaultDataPackagingServiceTest.java index fa87b177..4365589a 100644 --- a/src/test/java/gov/nist/oar/distrib/service/DefaultDataPackagingServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/DefaultDataPackagingServiceTest.java @@ -77,7 +77,7 @@ public static void createRequest() throws JsonParseException, JsonMappingExcepti FileRequest testval2 = mapper.readValue(val2, FileRequest.class); requestedUrls[0] = testval1; requestedUrls[1] = testval2; - bundleRequest = new BundleRequest("testdatabundle", requestedUrls); + bundleRequest = new BundleRequest("testdatabundle", requestedUrls,0); } @BeforeClass @@ -168,7 +168,7 @@ public void testBundleWithWarnings() FileRequest fileReq2 = mapper.readValue(file2, FileRequest.class); rUrls[0] = fileReq1; rUrls[1] = fileReq2; - bRequest = new BundleRequest("testdatabundle", rUrls); + bRequest = new BundleRequest("testdatabundle", rUrls,0); ddpkService = new DefaultDataPackagingService(domains, maxFSize, nOfFiles, redirectURLTrials); Path path = Files.createTempFile("testdatabundle", ".zip"); OutputStream os = Files.newOutputStream(path); diff --git a/src/test/java/gov/nist/oar/distrib/web/BundleDownloadPlanControllerTest.java b/src/test/java/gov/nist/oar/distrib/web/BundleDownloadPlanControllerTest.java index 79a2d7e4..1dc1968b 100644 --- a/src/test/java/gov/nist/oar/distrib/web/BundleDownloadPlanControllerTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/BundleDownloadPlanControllerTest.java @@ -86,7 +86,7 @@ public void testBundlePlanWithWarnings() FileRequest testval2 = mapper.readValue(val2, FileRequest.class); inputfileList[0] = testval1; inputfileList[1] = testval2; - BundleRequest bFL = new BundleRequest("testdownload-3", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload-3", inputfileList,0); RequestEntity request = RequestEntity.post(new URI(getBaseURL() + "/ds/_bundle_plan")) .body(bFL); @@ -114,7 +114,7 @@ public void testBundlePlanWithExceptions() FileRequest[] inputfileList = new FileRequest[3]; String val1 = "{\"filePath\":\"/1894/license.pdf\",\"downloadUrl\":\"https://s3.amazonaws.com/nist-midas/1894/license.pdf\"}"; String val2 = "{\"filePath\":\"/1895/license2.pdf\",\"downloadUrl\":\"https://s3.amazonaws.com/nist-midas/1894/license.pdf\"}"; - String val3 = "{\"filePath\":\"/1896/license3.pdfTEST Erro Here\",\"downloadUrl\":\"https://s3.amazonaws.com/nist-midas/1894/license.pdf\"}"; + String val3 = "{\"filePath\":\"/1896/license3.pdfTEST Error Here\",\"downloadUrl\":\"https://s3.amazonaws.com/nist-midas/1894/license.pdf\"}"; ObjectMapper mapper = new ObjectMapper(); FileRequest testval1 = mapper.readValue(val1, FileRequest.class); @@ -124,10 +124,12 @@ public void testBundlePlanWithExceptions() inputfileList[1] = testval2; inputfileList[2] = testval3; - BundleRequest bFL = new BundleRequest("testdownload-4", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload-4", inputfileList,99); RequestEntity request = RequestEntity.post(new URI(getBaseURL() + "/ds/_bundle_plan")) .body(bFL); - + System.out.println("request.getStatusCode():" + request + " \n request.getHeaders() :" + + request.getHeaders() + "\n request.getBody():" + request.getBody()); + ResponseEntity response = websvc.exchange(request, String.class); System.out.println("response.getStatusCode():" + response.getStatusCode() + " \n resp.getHeaders() :" + response.getHeaders() + "\n resp.getBody().length():" + response.getBody()); @@ -160,7 +162,7 @@ public void testBundlePlanWithErrors() ifileList[1] = fileRequest2; - BundleRequest bundleRequest = new BundleRequest("testdownload-5", ifileList); + BundleRequest bundleRequest = new BundleRequest("testdownload-5", ifileList,0); RequestEntity newRequest = RequestEntity.post(new URI(getBaseURL() + "/ds/_bundle_plan")) .body(bundleRequest); @@ -199,7 +201,7 @@ public void testBundlePlanWithNewWarnings() inputfileList[1] = testval2; inputfileList[2] = testval3; - BundleRequest bFL = new BundleRequest("testdownload-6", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload-6", inputfileList,0); RequestEntity request = RequestEntity.post(new URI(getBaseURL() + "/ds/_bundle_plan")) .body(bFL); diff --git a/src/test/java/gov/nist/oar/distrib/web/DataBundleAccessControllerTest.java b/src/test/java/gov/nist/oar/distrib/web/DataBundleAccessControllerTest.java index 354a9974..148fab45 100644 --- a/src/test/java/gov/nist/oar/distrib/web/DataBundleAccessControllerTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/DataBundleAccessControllerTest.java @@ -89,7 +89,7 @@ public void testDownloadAllFiles() FileRequest testval2 = mapper.readValue(val2, FileRequest.class); inputfileList[0] = testval1; inputfileList[1] = testval2; - BundleRequest bFL = new BundleRequest("testdownload-1", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload-1", inputfileList,0); RequestEntity request = RequestEntity.post(new URI(getBaseURL() + "/ds/_bundle")).body(bFL); ResponseEntity response = websvc.exchange(request, String.class); @@ -118,7 +118,7 @@ public void testDownloadAllFilesException() inputfileList[1] = testval2; inputfileList[2] = testval3; - BundleRequest bFL = new BundleRequest("testdownload-2", inputfileList); + BundleRequest bFL = new BundleRequest("testdownload-2", inputfileList,0); RequestEntity request = RequestEntity.post(new URI(getBaseURL() + "/ds/_bundle")).body(bFL); ResponseEntity response = websvc.exchange(request, String.class);