From 422b132f84e3e53517035887d902d57eff401ebd Mon Sep 17 00:00:00 2001 From: Berend Weel Date: Thu, 2 Nov 2017 14:37:12 +0100 Subject: [PATCH] Updated to Xenon 2.1.0 --- build.gradle | 7 ++- .../config/ComputeResource.java | 2 +- .../rest/Swagger2SpringBoot.java | 32 ---------- .../rest/service/JobService.java | 1 - .../rest/service/XenonService.java | 61 +++++++++++++------ .../rest/service/tasks/CwlStageInTask.java | 1 - .../rest/service/tasks/DeleteJobTask.java | 3 - 7 files changed, 48 insertions(+), 59 deletions(-) diff --git a/build.gradle b/build.gradle index d46abf9..9ac140c 100644 --- a/build.gradle +++ b/build.gradle @@ -8,6 +8,10 @@ plugins { id 'org.springframework.boot' version '1.5.6.RELEASE' } +springBoot { + mainClass = 'nl.esciencecenter.computeservice.rest.Swagger2SpringBoot' +} + bootRepackage { mainClass = 'nl.esciencecenter.computeservice.rest.Swagger2SpringBoot' } @@ -24,8 +28,7 @@ repositories { // In this section you declare the dependencies for your production and test code dependencies { // Xenon - compile "com.github.NLeSC:Xenon:master-SNAPSHOT" - //compile group: 'nl.esciencecenter.xenon', name: 'xenon', version: '2.0.0' + compile group: 'nl.esciencecenter.xenon', name: 'xenon', version: '2.1.0' //Spring(boot) compile group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '1.5.6.RELEASE' diff --git a/src/main/java/nl/esciencecenter/computeservice/config/ComputeResource.java b/src/main/java/nl/esciencecenter/computeservice/config/ComputeResource.java index 16a5cfd..aa87e14 100644 --- a/src/main/java/nl/esciencecenter/computeservice/config/ComputeResource.java +++ b/src/main/java/nl/esciencecenter/computeservice/config/ComputeResource.java @@ -22,7 +22,7 @@ public class ComputeResource { @JsonProperty("scheduler") private AdaptorConfig schedulerConfig; - @JsonProperty("filesystem") + @JsonProperty(value="filesystem", required=false) private AdaptorConfig filesystemConfig; @JsonProperty(value="cwlCommand", required=false) diff --git a/src/main/java/nl/esciencecenter/computeservice/rest/Swagger2SpringBoot.java b/src/main/java/nl/esciencecenter/computeservice/rest/Swagger2SpringBoot.java index edf902b..ada3edd 100644 --- a/src/main/java/nl/esciencecenter/computeservice/rest/Swagger2SpringBoot.java +++ b/src/main/java/nl/esciencecenter/computeservice/rest/Swagger2SpringBoot.java @@ -1,11 +1,5 @@ package nl.esciencecenter.computeservice.rest; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.Map; - -import org.apache.catalina.servlets.WebdavServlet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -15,8 +9,6 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.embedded.EmbeddedServletContainerInitializedEvent; -import org.springframework.boot.web.servlet.FilterRegistrationBean; -import org.springframework.boot.web.servlet.ServletRegistrationBean; import org.springframework.context.ApplicationListener; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -24,9 +16,6 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.springframework.web.cors.CorsConfiguration; -import org.springframework.web.cors.UrlBasedCorsConfigurationSource; -import org.springframework.web.filter.CorsFilter; import springfox.documentation.swagger2.annotations.EnableSwagger2; @@ -40,27 +29,6 @@ "nl.esciencecenter.computeservice.cwl.*" }) public class Swagger2SpringBoot implements CommandLineRunner, ApplicationListener { private static final Logger logger = LoggerFactory.getLogger(Swagger2SpringBoot.class); - -// I can't get the CORS for the embedded webdav to work, so it's not much sense -// to include it. -// @Bean -// public static ServletRegistrationBean servletRegistrationBean() { -// -// final ServletRegistrationBean registration = new ServletRegistrationBean(new WebdavServlet()); -// -// final Map params = new HashMap(); -// params.put("debug", "1"); -// params.put("listings", "true"); -// params.put("readonly", "false"); -// -// params.put("cors.allowed.origins", "*"); -// -// registration.setInitParameters(params); -// registration.addUrlMappings("/webdav/*"); -// registration.setLoadOnStartup(1); -// -// return registration; -// } @Value("${server.address}") String bindAdress; diff --git a/src/main/java/nl/esciencecenter/computeservice/rest/service/JobService.java b/src/main/java/nl/esciencecenter/computeservice/rest/service/JobService.java index d29daac..96a6f05 100644 --- a/src/main/java/nl/esciencecenter/computeservice/rest/service/JobService.java +++ b/src/main/java/nl/esciencecenter/computeservice/rest/service/JobService.java @@ -14,7 +14,6 @@ import nl.esciencecenter.computeservice.rest.model.JobState; import nl.esciencecenter.computeservice.rest.model.StatePreconditionException; import nl.esciencecenter.computeservice.rest.model.WorkflowBinding; -import nl.esciencecenter.xenon.XenonException; import nl.esciencecenter.xenon.filesystems.Path; @Service diff --git a/src/main/java/nl/esciencecenter/computeservice/rest/service/XenonService.java b/src/main/java/nl/esciencecenter/computeservice/rest/service/XenonService.java index 3ed3d6d..d0f10e7 100644 --- a/src/main/java/nl/esciencecenter/computeservice/rest/service/XenonService.java +++ b/src/main/java/nl/esciencecenter/computeservice/rest/service/XenonService.java @@ -101,7 +101,6 @@ private void initialize() throws XenonException, IOException { // TODO: Is assertions the nicest way? assert(resource != null); assert(resource.getSchedulerConfig() != null); - assert(resource.getFilesystemConfig() != null); // Start up submitted jobs @@ -170,18 +169,51 @@ public JobService getJobService() { public void setJobService(JobService jobService) { this.jobService = jobService; } - - public Scheduler getScheduler() throws XenonException { - if (scheduler == null) { - // Initialize xenon scheduler - ComputeResource resource = getConfig().defaultComputeResource(); - AdaptorConfig schedulerConfig = resource.getSchedulerConfig(); - + + private void checkSchedulerStates() throws XenonException { + ComputeResource resource = getConfig().defaultComputeResource(); + AdaptorConfig schedulerConfig = resource.getSchedulerConfig(); + AdaptorConfig fileSystemConfig = resource.getFilesystemConfig(); + + boolean useSchedulerFilesystem = fileSystemConfig == null; + boolean recreateScheduler = false; + boolean recreateFileSystem = false; + if(useSchedulerFilesystem) { + if (scheduler == null || !scheduler.isOpen() || remoteFileSystem == null || !remoteFileSystem.isOpen()) { + recreateScheduler = true; + recreateFileSystem = true; + } + + } else { + if (scheduler == null || !scheduler.isOpen()) { + recreateScheduler = true; + } + if (remoteFileSystem == null || !remoteFileSystem.isOpen()) { + recreateFileSystem = true; + } + } + if (recreateScheduler) { logger.debug("Creating a scheduler to run jobs..."); scheduler = Scheduler.create(schedulerConfig.getAdaptor(), schedulerConfig.getLocation(), - schedulerConfig.getCredential(), schedulerConfig.getProperties()); + schedulerConfig.getCredential(), schedulerConfig.getProperties()); + } + if (recreateFileSystem) { + if (useSchedulerFilesystem && Scheduler.getAdaptorDescription(scheduler.getAdaptorName()).usesFileSystem()) { + logger.debug("Using scheduler filesystem as a remote filesystem..."); + remoteFileSystem = scheduler.getFileSystem(); + } else { + // Initialize remote filesystem + logger.debug("Creating remote filesystem..."); + remoteFileSystem = FileSystem.create(fileSystemConfig.getAdaptor(), fileSystemConfig.getLocation(), + fileSystemConfig.getCredential(), fileSystemConfig.getProperties()); + + logger.debug("Remote working directory: " + remoteFileSystem.getWorkingDirectory()); + } } + } + public Scheduler getScheduler() throws XenonException { + checkSchedulerStates(); return scheduler; } @@ -197,16 +229,7 @@ public void setScheduler(Scheduler scheduler) { } public FileSystem getRemoteFileSystem() throws XenonException { - if (remoteFileSystem == null || !remoteFileSystem.isOpen()) { - // Initialize remote filesystem - logger.debug("Creating remote filesystem..."); - ComputeResource resource = getConfig().defaultComputeResource(); - AdaptorConfig fileSystemConfig = resource.getFilesystemConfig(); - remoteFileSystem = FileSystem.create(fileSystemConfig.getAdaptor(), fileSystemConfig.getLocation(), - fileSystemConfig.getCredential(), fileSystemConfig.getProperties()); - - logger.debug("Remote working directory: " + remoteFileSystem.getWorkingDirectory()); - } + checkSchedulerStates(); return remoteFileSystem; } diff --git a/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/CwlStageInTask.java b/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/CwlStageInTask.java index 9cd63ac..9977372 100644 --- a/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/CwlStageInTask.java +++ b/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/CwlStageInTask.java @@ -25,7 +25,6 @@ import nl.esciencecenter.computeservice.rest.service.staging.StringToFileStagingObject; import nl.esciencecenter.computeservice.rest.service.staging.XenonStager; import nl.esciencecenter.xenon.XenonException; -import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; diff --git a/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/DeleteJobTask.java b/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/DeleteJobTask.java index 9187399..f92e94e 100644 --- a/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/DeleteJobTask.java +++ b/src/main/java/nl/esciencecenter/computeservice/rest/service/tasks/DeleteJobTask.java @@ -5,7 +5,6 @@ import nl.esciencecenter.computeservice.rest.model.Job; import nl.esciencecenter.computeservice.rest.model.JobRepository; -import nl.esciencecenter.computeservice.rest.service.JobService; import nl.esciencecenter.computeservice.rest.service.XenonService; import nl.esciencecenter.xenon.XenonException; import nl.esciencecenter.xenon.filesystems.FileSystem; @@ -17,13 +16,11 @@ public class DeleteJobTask implements Runnable { private String jobId; private XenonService service; private JobRepository repository; - private JobService jobService; public DeleteJobTask(String jobId, XenonService service) throws XenonException { this.jobId = jobId; this.service = service; this.repository = service.getRepository(); - this.jobService = service.getJobService(); } @Override