988 n5 export rpc

docker/build/Dockerfile-data-dev

@@ -59,7 +59,8 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \
     simdatadir_external=/path/to/external/simdata/ \
     simdataCacheSize="simdataCacheSize-not-set" \
     webDataPort="webDataPort-not-set" \
-    servertype="servertype-not-set"
+    servertype="servertype-not-set" \
+    s3ProxyPortExternal="80"
 
 ENV dbpswdfile=/run/secrets/dbpswd \
     jmspswdfile=/run/secrets/jmspswd \
@@ -89,6 +90,8 @@ ENTRYPOINT java \
 	-Dvcell.python.executable=/usr/local/bin/python \
 	-Dvcell.primarySimdatadir.internal=/simdata \
 	-Dvcell.secondarySimdatadir.internal=/simdata_secondary \
+	-Dvcell.n5DataDir.internal=/n5DataDir \
+	-Dvcell.s3proxy.port.external="${s3ProxyPortExternal}" \
 	-Dvcell.primarySimdatadir.external="${simdatadir_external}" \
 	-Dvcell.simdataCacheSize="${simdataCacheSize}" \
 	-Dvcell.export.baseDir.internal=/exportdir/ \

docker/swarm/docker-compose.yml

@@ -269,6 +269,7 @@ services:
       - mongodb_host_internal=mongodb
       - mongodb_port_internal=27017
       - mongodb_database=test
+      - s3ProxyPortExternal=${VCELL_S3PROXY_PORT_EXTERNAL}
 
       - webDataPort=${VCELL_WEB_DATA_PORT}
 

docker/swarm/localconfig_mockslurm-not-used.sh

@@ -100,6 +100,7 @@ VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset))
 VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset))
 VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset))
 VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset))
+VCELL_S3PROXY_PORT_EXTERNAL=$((8086 + _site_port_offset))
 VCELL_HTC_NODELIST=
 VCELL_BATCH_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-batch:${VCELL_TAG}"
 VCELL_BATCH_SINGULARITY_FILENAME="${VCELL_BATCH_DOCKER_IMAGE//[\/:]/_}.img"
@@ -159,6 +160,8 @@ VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME
 VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL
 VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE
 VCELL_SECRETS_DIR=${HOME}/vcellkeys
+VCELL_S3PROXY_PORT_EXTERNAL=$VCELL_S3PROXY_PORT_EXTERNAL
+VCELL_N5_DATADIR_HOST=/share/apps/vcell3/n5
 VCELL_SIMDATADIR_EXTERNAL=${VCELL_DEPLOY_ROOT_DIR}/users
 VCELL_SIMDATADIR_HOST=${VCELL_DEPLOY_ROOT_DIR}/users
 VCELL_SIMDATADIR_SECONDARY_HOST=/Volumes/vcell/users

docker/swarm/localconfig_realslurm_oracle_zeke.sh

@@ -119,8 +119,8 @@ VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCEL
 VCELL_SMTP_HOSTNAME=vdsmtp.cam.uchc.edu
 VCELL_SMTP_PORT=25
 [email protected]
-VCELL_EXPORT_BASEURL=http://vcell.org/export/
-VCELL_EXPORTDIR_HOST=/Volumes/vcell/export/
+VCELL_EXPORT_BASEURL=http://localhost/
+VCELL_EXPORTDIR_HOST=/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage/export
 VCELL_MAX_JOBS_PER_SCAN=100
 VCELL_MAX_ODE_JOBS_PER_USER=100
 VCELL_MAX_PDE_JOBS_PER_USER=40
@@ -136,65 +136,70 @@ VCELL_SSH_CMD_RESTORE_TIMEOUT=5
 cat <<EOF >"$_outputfile"
 VCELL_API_HOST_EXTERNAL=$VCELL_API_HOST_EXTERNAL
 VCELL_API_PORT_EXTERNAL=$VCELL_API_PORT_EXTERNAL
+
 VCELL_BATCH_DOCKER_IMAGE=$VCELL_BATCH_DOCKER_IMAGE
 VCELL_BATCH_HOST=$VCELL_BATCH_HOST
 VCELL_BATCH_SINGULARITY_FILENAME=$VCELL_BATCH_SINGULARITY_FILENAME
 VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL=$VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL
 VCELL_BATCH_USER=vcell
+
 VCELL_BIOFORMATS_JAR_FILE=vcell-bioformats-0.0.9-jar-with-dependencies.jar
 VCELL_BIOFORMATS_JAR_URL=http://vcell.org/webstart/vcell-bioformats-0.0.9-jar-with-dependencies.jar
+
 VCELL_BUILD_NUMBER=$VCELL_BUILD_NUMBER
 VCELL_CLIENT_APPID=${_applicationId}
+
 VCELL_DB_DRIVER=$VCELL_DB_DRIVER
 VCELL_DB_URL=$VCELL_DB_URL
 VCELL_DB_USER=$VCELL_DB_USER
+
 VCELL_DEBUG_PORT_VCELL_API=$((VCELL_DEBUG_PORT_BASE+1))
 VCELL_DEBUG_PORT_VCELL_DATA=$((VCELL_DEBUG_PORT_BASE+2))
 VCELL_DEBUG_PORT_VCELL_DB=$((VCELL_DEBUG_PORT_BASE+3))
 VCELL_DEBUG_PORT_VCELL_SCHED=$((VCELL_DEBUG_PORT_BASE+4))
 VCELL_DEBUG_PORT_VCELL_SUBMIT=$((VCELL_DEBUG_PORT_BASE+5))
 VCELL_DEBUG_PORT_VCELL_WEB=$((VCELL_DEBUG_PORT_BASE+6))
-VCELL_DEPLOY_SECRETS_DIR=${HOME}/vcellkeys_oracle
+
+VCELL_DEPLOY_SECRETS_DIR=/media/zeke/DiskDrive/Home/Work/CCAM/VCellDummyFiles
 VCELL_EXPORT_BASEURL=$VCELL_EXPORT_BASEURL
 VCELL_EXPORTDIR_HOST=$VCELL_EXPORTDIR_HOST
 VCELL_HTC_NODELIST=$VCELL_HTC_NODELIST
 VCELL_HTCLOGS_EXTERNAL=/share/apps/vcell3/dev_htclogs
-VCELL_HTCLOGS_HOST=/Volumes/vcell/dev_htclogs
-VCELL_INSTALLER_JRE_LINUX32=linux-x86-1.8.0_66
-VCELL_INSTALLER_JRE_LINUX64=linux-amd64-1.8.0_66
-VCELL_INSTALLER_JRE_MAC=macosx-amd64-1.8.0_141
-VCELL_INSTALLER_JRE_WIN32=windows-x86-1.8.0_141
-VCELL_INSTALLER_JRE_WIN64=windows-amd64-1.8.0_141
-VCELL_INSTALLER_JREDIR=${HOME}/.install4j6/jres
+VCELL_HTCLOGS_HOST=/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage/dev_htclogs
+
 VCELL_JMS_SIM_HOST_EXTERNAL=$VCELL_JMS_SIM_HOST_EXTERNAL
 VCELL_JMS_SIM_PORT_EXTERNAL=$VCELL_JMS_SIM_PORT_EXTERNAL
 VCELL_JMS_SIM_RESTPORT_EXTERNAL=$VCELL_JMS_SIM_RESTPORT_EXTERNAL
+
 VCELL_MAX_JOBS_PER_SCAN=$VCELL_MAX_JOBS_PER_SCAN
 VCELL_MAX_ODE_JOBS_PER_USER=$VCELL_MAX_ODE_JOBS_PER_USER
 VCELL_MAX_PDE_JOBS_PER_USER=$VCELL_MAX_PDE_JOBS_PER_USER
+
 VCELL_MONGO_HOST_EXTERNAL=$VCELL_MONGO_HOST_EXTERNAL
 VCELL_MONGO_PORT_EXTERNAL=$VCELL_MONGO_PORT_EXTERNAL
+
 VCELL_NATIVESOLVERDIR_EXTERNAL=/share/apps/vcell3/nativesolvers
 VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE
 VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME
 VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL
+
 VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE
-VCELL_SECRETS_DIR=${HOME}/vcellkeys_oracle
-VCELL_N5_DATADIR_HOST=/share/apps/vcell3/n5
+VCELL_SECRETS_DIR=/media/zeke/DiskDrive/Home/Work/CCAM/VCellDummyFiles
+VCELL_N5_DATADIR_HOST=/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage/N5Dir
 VCELL_S3PROXY_PORT_EXTERNAL=$VCELL_S3PROXY_PORT_EXTERNAL
+
 VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users
 VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users
-VCELL_SIMDATADIR_ARCHIVE_HOST=/Volumes/vcell-1/users
-VCELL_SIMDATADIR_ARCHIVE_HOST_ORIG=/Volumes/vcell-2/users
+VCELL_SIMDATADIR_ARCHIVE_HOST=/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage/users
 VCELL_SIMDATADIR_EXTERNAL=/share/apps/vcell3/users
-VCELL_SIMDATADIR_HOST=/Volumes/vcell/users
+VCELL_SIMDATADIR_HOST=/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage/users
 VCELL_SIMDATADIR_SECONDARY_EXTERNAL=/share/apps/vcell3/users
-VCELL_SIMDATADIR_SECONDARY_HOST=/Volumes/vcell/users
-VCELL_SIMDATADIR_SECONDARY_EXTERNAL_ORIG=/share/apps/vcell7/users
-VCELL_SIMDATADIR_SECONDARY_HOST_ORIG=/Volumes/vcell-1/users
+VCELL_SIMDATADIR_SECONDARY_HOST=~/Mnts/RemoteVolumes/VCell05/users
 VCELL_SIMDATADIR_PARALLEL_EXTERNAL=/share/apps/vcell3parallel
+
 VCELL_SITE_CAMEL=${_site_camel}
 VCELL_SITE=$VCELL_SITE
+
 VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR}
 VCELL_SLURM_CMD_SACCT=$VCELL_SLURM_CMD_SACCT
 VCELL_SLURM_CMD_SBATCH=$VCELL_SLURM_CMD_SBATCH
@@ -208,16 +213,21 @@ VCELL_SLURM_QOS=$VCELL_SLURM_QOS
 VCELL_SLURM_RESERVATION_PU=$VCELL_SLURM_RESERVATION_PU
 VCELL_SLURM_RESERVATION=$VCELL_SLURM_RESERVATION
 VCELL_SLURM_TMPDIR=$VCELL_SLURM_TMPDIR
+
 VCELL_SMTP_EMAILADDRESS=${VCELL_SMTP_EMAILADDRESS}
 VCELL_SMTP_HOSTNAME=${VCELL_SMTP_HOSTNAME}
 VCELL_SMTP_PORT=${VCELL_SMTP_PORT}
+
 VCELL_SSH_CMD_RESTORE_TIMEOUT=$VCELL_SSH_CMD_RESTORE_TIMEOUT
 VCELL_SSH_CMD_TIMEOUT=$VCELL_SSH_CMD_TIMEOUT
+
 VCELL_SSL_IGNORE_CERT_PROBLEMS=true
 VCELL_SSL_IGNORE_HOST_MISMATCH=true
+
 VCELL_SUBMIT_SERVICE_HOST=submit
 VCELL_TAG=$VCELL_TAG
 VCELL_UPDATE_SITE=http://vcell.org/webstart/${_site_camel}
+
 VCELL_VERSION_NUMBER=$VCELL_VERSION_NUMBER
 VCELL_VERSION=${_site_camel}_Version_${VCELL_VERSION_NUMBER}_build_${VCELL_BUILD_NUMBER}
 VCELL_WEB_DATA_PORT=$VCELL_WEB_DATA_PORT

vcell-api/src/main/java/org/vcell/rest/n5data/N5GetInfoRestlet.java

@@ -45,16 +45,17 @@ public void handle(Request req, Response response) {
 
 				switch (desiredInfo){
 					case VCellApiApplication.N5_INFO_SUPPORTED_SPECIES:
-						User user = application.getVCellUser(req.getChallengeResponse(), VCellApiApplication.AuthenticationPolicy.ignoreInvalidCredentials);
-						N5Service n5Service = new N5Service(simID, user);
-						ArrayList<String> listOfSupportedSpecies = n5Service.supportedSpecies();
-						Gson gson = new Gson();
-						response.setEntity(gson.toJson(listOfSupportedSpecies), MediaType.APPLICATION_JSON);
+//						User user = application.getVCellUser(req.getChallengeResponse(), VCellApiApplication.AuthenticationPolicy.ignoreInvalidCredentials);
+//						N5Service n5Service = new N5Service(simID, user);
+//						ArrayList<String> listOfSupportedSpecies = n5Service.supportedSpecies();
+//						Gson gson = new Gson();
+//						response.setEntity(gson.toJson(listOfSupportedSpecies), MediaType.APPLICATION_JSON);
 						return;
 					default:
-						lg.info("No information type was specified in getting N5 info");
-						response.setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
-						response.setEntity("failed to specify type of N5 information request for sim ID: " + simID, MediaType.TEXT_PLAIN);
+						return;
+//						lg.info("No information type was specified in getting N5 info");
+//						response.setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
+//						response.setEntity("failed to specify type of N5 information request for sim ID: " + simID, MediaType.TEXT_PLAIN);
 				}
 			} catch (Exception e) {
 				lg.error(e.getMessage(), e);

vcell-api/src/main/java/org/vcell/rest/n5data/N5Service.java

@@ -3,7 +3,6 @@
 import cbit.vcell.math.MathException;
 import cbit.vcell.resource.PropertyLoader;
 import cbit.vcell.simdata.DataIdentifier;
-import cbit.vcell.simdata.n5.N5Exporter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.janelia.saalfeldlab.n5.Compression;
@@ -38,21 +37,20 @@ public N5Service(String simID, User user) {
 
 
     public void exportToN5(String[] species, Compression compression) throws IOException, DataAccessException, MathException {
-        N5Exporter n5Exporter = new N5Exporter();
-        n5Exporter.initalizeDataControllers(simID, user.getName(), user.getID().toString());
-        ArrayList<DataIdentifier> dataIdentifiers = new ArrayList<>();
-        for(String specie: species){
-            dataIdentifiers.add(n5Exporter.getSpecificDI(specie));
-        }
-        n5Exporter.exportToN5(dataIdentifiers, compression);
+//        N5Exporter n5Exporter = new N5Exporter();
+//        n5Exporter.initalizeDataControllers(simID, user.getName(), user.getID().toString());
+//        ArrayList<DataIdentifier> dataIdentifiers = new ArrayList<>();
+//        for(String specie: species){
+//            dataIdentifiers.add(n5Exporter.getSpecificDI(specie));
+//        }
+//        n5Exporter.exportToN5(dataIdentifiers, compression);
+        return;
     }
 
 
-    public ArrayList<String> supportedSpecies() throws IOException, DataAccessException {
-        N5Exporter n5Exporter = new N5Exporter();
-        n5Exporter.initalizeDataControllers(simID, user.getName(), user.getID().toString());
-        return n5Exporter.getSupportedSpecies();
-    }
+//    public ArrayList<String> supportedSpecies() throws IOException, DataAccessException {
+//
+//    }
 
 
 

vcell-client/src/main/java/cbit/vcell/client/ClientRequestManager.java

@@ -68,6 +68,7 @@
 import javax.swing.Timer;
 import javax.swing.filechooser.FileFilter;
 
+import cbit.vcell.export.server.N5Specs;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
@@ -2540,6 +2541,9 @@ public static void downloadExportedData(final Component requester, final UserPre
 		if (exportsRecord.contains(evt.getJobID() + "")) {
 			return;
 		}
+		if (evt.getLocation().toLowerCase().endsWith("." + N5Specs.n5Suffix)){
+			return;
+		}
 		AsynchClientTask task1 = new AsynchClientTask("Retrieving data from '" + evt.getLocation() + "'",
 				AsynchClientTask.TASKTYPE_NONSWING_BLOCKING) {