Skip to content

Commit

Permalink
[simple-api] fixes for usage through jni
Browse files Browse the repository at this point in the history 
- Handle URIs in options
- Return the job object for external monitoring
- Generalize withArgument to allow the use of java objects
and raw types instead of string in options
- handle null arguments
- when writing result, decode the id to avoid url-encoded file names
  • Loading branch information
@NPavie
NPavie committed Nov 20, 2023
1 parent e8bb3bd commit 35c0c97
Showing 1 changed file with 104 additions and 80 deletions.
184 changes: 104 additions & 80 deletions src/main/java/SimpleAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.concurrent.atomic.AtomicBoolean;
Expand Down Expand Up @@ -35,65 +38,72 @@
import org.osgi.service.component.annotations.ReferencePolicy;

/**
* A simplified Java API consisting of a {@link #startJob()} method that starts a job based on a
* A simplified Java API consisting of a {@link #startJob(String, Map<String,Object>)} method that starts a job based on a
* script name and a list of options, and {@link #getNewMessages()} and {@link #getLastJobStatus()}
* methods. It is used to build a simple Java CLI (see the {@link #main()} method). The simplified
* methods. It is used to build a simple Java CLI (see the {@link #main(String[])} method). The simplified
* API also makes it easier to bridge with other programming languages using JNI.
*/
@Component(
name = "SimpleAPI",
immediate = true
name = "SimpleAPI",
immediate = true
)
public class SimpleAPI {

private ScriptRegistry scriptRegistry;
private JobFactory jobFactory;

@Reference(
name = "script-registry",
unbind = "-",
service = ScriptRegistry.class,
cardinality = ReferenceCardinality.MANDATORY,
policy = ReferencePolicy.STATIC
name = "script-registry",
unbind = "-",
service = ScriptRegistry.class,
cardinality = ReferenceCardinality.MANDATORY,
policy = ReferencePolicy.STATIC
)
public void setScriptRegistry(ScriptRegistry scriptRegistry) {
this.scriptRegistry = scriptRegistry;
}

@Reference(
name = "job-factory",
unbind = "-",
service = JobFactory.class,
cardinality = ReferenceCardinality.MANDATORY,
policy = ReferencePolicy.STATIC
name = "job-factory",
unbind = "-",
service = JobFactory.class,
cardinality = ReferenceCardinality.MANDATORY,
policy = ReferencePolicy.STATIC
)
public void setJobFactory(JobFactory jobFactory) {
this.jobFactory = jobFactory;
}

private void _startJob(String scriptName, Map<String,? extends Iterable<String>> options) throws IllegalArgumentException, FileNotFoundException {
private CommandLineJob _startJob(String scriptName, Map<String, Object> options) throws IllegalArgumentException, FileNotFoundException, URISyntaxException {
ScriptService<?> scriptService = scriptRegistry.getScript(scriptName);
if (scriptService == null)
throw new IllegalArgumentException(scriptName + " script not found");
Script script = scriptService.load();
File fileBase = new File(System.getProperty("org.daisy.pipeline.cli.cwd", "."));
CommandLineJobParser parser = new CommandLineJobParser(script, fileBase);
for (Map.Entry<String,? extends Iterable<String>> e : options.entrySet())
for (String value : e.getValue())
for (Map.Entry<String,Object> e : options.entrySet()){
Object value = e.getValue();
if(value instanceof List){
for (Object subValue : (List<Object>)value)
parser.withArgument(e.getKey(), subValue);
} else {
parser.withArgument(e.getKey(), value);
}
}
CommandLineJob job = parser.createJob(jobFactory);
MessageAccessor accessor = job.getMonitor().getMessageAccessor();
accessor.listen(
num -> {
consumeMessage(accessor, num);
}
num -> {
consumeMessage(accessor, num);
}
);
job.getMonitor().getStatusUpdates().listen(s -> updateJobStatus(s));
new Thread(job).start();
return job;
}

public static void startJob(String scriptName, Map<String,? extends Iterable<String>> options) throws IllegalArgumentException, FileNotFoundException {
getInstance()._startJob(scriptName, options);
public static CommandLineJob startJob(String scriptName, Map<String,Object> options) throws IllegalArgumentException, FileNotFoundException, URISyntaxException {
return getInstance()._startJob(scriptName, options);
}

/**
Expand All @@ -120,9 +130,9 @@ private static synchronized void updateJobStatus(Job.Status status) {
private static synchronized void consumeMessage(MessageAccessor accessor, int seqNum) {
for (Message m :
accessor.createFilter()
.greaterThan(lastMessage)
.filterLevels(Collections.singleton(Level.INFO))
.getMessages()) {
.greaterThan(lastMessage)
.filterLevels(Collections.singleton(Level.INFO))
.getMessages()) {
if (m.getSequence() > lastMessage) {
messagesQueue.add(m);
}
Expand Down Expand Up @@ -153,7 +163,7 @@ public static void main(String[] args) throws InterruptedException, IOException
System.exit(1);
}
String script = args[0];
Map<String,List<String>> options = new HashMap<>();
Map<String,Object> options = new HashMap<>();
for (int i = 1; i < args.length; i += 2) {
if (!args[i].startsWith("--")) {
System.err.println("Expected option name argument, got " + args[i]);
Expand All @@ -164,7 +174,7 @@ public static void main(String[] args) throws InterruptedException, IOException
System.err.println("Expected option value argument");
System.exit(1);
}
List<String> list = options.get(option);
List<String> list = (List<String>)options.get(option);
if (list == null) {
list = new ArrayList<>();
options.put(option, list);
Expand All @@ -176,7 +186,7 @@ public static void main(String[] args) throws InterruptedException, IOException
} catch (IllegalArgumentException e) {
System.err.println(e.getMessage());
System.exit(1);
} catch (FileNotFoundException e) {
} catch (FileNotFoundException | URISyntaxException e) {
System.err.println("File does not exist: " + e.getMessage());
System.exit(1);
}
Expand All @@ -185,14 +195,14 @@ public static void main(String[] args) throws InterruptedException, IOException
System.err.println(m.getText());
}
switch (SimpleAPI.getLastJobStatus()) {
case SUCCESS:
case FAIL:
case ERROR:
System.exit(0);
case IDLE:
case RUNNING:
default:
Thread.sleep(1000);
case SUCCESS:
case FAIL:
case ERROR:
System.exit(0);
case IDLE:
case RUNNING:
default:
Thread.sleep(1000);
}
}
}
Expand All @@ -214,13 +224,14 @@ public CommandLineJobParser(Script script, File fileBase) {
/**
* Parse command line argument
*/
public CommandLineJobParser withArgument(String key, String value) throws IllegalArgumentException, FileNotFoundException {
public CommandLineJobParser withArgument(String key, Object value) throws IllegalArgumentException, FileNotFoundException, URISyntaxException {
if(value == null) return this;
if (script.getInputPort(key) != null)
return withInput(key, value);
return withInput(key, value.toString());
else if (script.getOption(key) != null)
return withOption(key, value);
else if (script.getOutputPort(key) != null)
return withOutput(key, value);
return withOutput(key, value.toString());
else
throw new IllegalArgumentException("Unknown argument: " + key);
}
Expand Down Expand Up @@ -257,12 +268,19 @@ private CommandLineJobParser withInput(String port, String source) throws Illega
* @throws FileNotFoundException if the option type is "anyFileURI" and the value can not be
* resolved to a document.
*/
private CommandLineJobParser withOption(String name, String value) throws IllegalArgumentException, FileNotFoundException {
private CommandLineJobParser withOption(String name, Object value) throws IllegalArgumentException, FileNotFoundException, URISyntaxException {
ScriptOption o = script.getOption(name);
if (o != null) {
String type = o.getType().getId();
if ("anyFileURI".equals(type)) {
File file = new File(value);
String pathValue = value.toString();

File file;
if(pathValue.startsWith("file:/")){
file = new File(new URI(pathValue));
} else {
file = new File(pathValue);
}
if (!file.isAbsolute()) {
if (fileBase == null)
throw new FileNotFoundException("File must be an absolute path, but got " + file);
Expand All @@ -274,7 +292,13 @@ private CommandLineJobParser withOption(String name, String value) throws Illega
throw new UncheckedIOException(e);
}
} else if ("anyDirURI".equals(type)) {
File dir = new File(value);
String pathValue = value.toString();
File dir;
if(pathValue.startsWith("file:/")){
dir = new File(new URI(pathValue));
} else {
dir = new File(pathValue);
}
if (!dir.isAbsolute()) {
if (fileBase == null)
throw new FileNotFoundException("File must be an absolute path, but got " + dir);
Expand All @@ -292,7 +316,7 @@ private CommandLineJobParser withOption(String name, String value) throws Illega
}
}
}
builder.withOption(name, value);
builder.withOption(name, value.toString());
return this;
}

Expand All @@ -306,10 +330,10 @@ private CommandLineJobParser withOutput(String port, String result) throws Illeg
ScriptPort p = script.getOutputPort(port);
if (p == null)
throw new IllegalArgumentException(
String.format("Output '%s' is not recognized by script '%s'", port, script.getId()));
String.format("Output '%s' is not recognized by script '%s'", port, script.getId()));
if (resultLocations.containsKey(port))
throw new IllegalArgumentException(
String.format("Output '%s' already specified", port));
String.format("Output '%s' already specified", port));
File file = new File(result);
if (!file.isAbsolute()) {
if (fileBase == null)
Expand All @@ -334,7 +358,7 @@ else if (file.list().length > 0)
if (file.isDirectory()) {
if (file.list().length > 0)
throw new IllegalArgumentException("Directory is not empty: " + file);
resultLocations.put(port, URI.create(file.toURI() + "/"));
resultLocations.put(port, file.toURI());
} else {
if (p.isSequence())
throw new IllegalArgumentException("Not a directory: " + file);
Expand Down Expand Up @@ -374,32 +398,32 @@ public void run() {
job.run();
try {
switch (job.getStatus()) {
case SUCCESS:
case FAIL:
List<File> existingFiles = new ArrayList<>();
for (String port : job.getResults().getPorts()) {
if (resultLocations.containsKey(port)) {
URI u = resultLocations.get(port);
File f = new File(u);
if (u.toString().endsWith("/"))
for (JobResult r : job.getResults().getResults(port)) {
File dest = new File(f, r.strip().getIdx());
if (dest.exists())
existingFiles.add(dest);
else
writeResult(r, dest);
}
else
for (JobResult r : job.getResults().getResults(port))
if (f.exists())
existingFiles.add(f);
else
writeResult(r, f);
case SUCCESS:
case FAIL:
List<File> existingFiles = new ArrayList<>();
for (String port : job.getResults().getPorts()) {
if (resultLocations.containsKey(port)) {
URI u = resultLocations.get(port);
File f = new File(u);
if (u.toString().endsWith("/"))
for (JobResult r : job.getResults().getResults(port)) {
File dest = new File(f, URLDecoder.decode(r.strip().getIdx(), StandardCharsets.UTF_8));
if (dest.exists())
existingFiles.add(dest);
else
writeResult(r, dest);
}
else
for (JobResult r : job.getResults().getResults(port))
if (f.exists())
existingFiles.add(f);
else
writeResult(r, f);
}
}
}
if (!existingFiles.isEmpty())
throw new IOException("Some results could not be written: " + existingFiles);
default:
if (!existingFiles.isEmpty())
throw new IOException("Some results could not be written: " + existingFiles);
default:
}
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand All @@ -410,14 +434,14 @@ public void run() {
public Job.Status getStatus() {
Job.Status s = job.getStatus();
switch (s) {
case SUCCESS:
case FAIL:
case ERROR:
return completed.get() ? s : Job.Status.RUNNING;
case IDLE:
case RUNNING:
default:
return s;
case SUCCESS:
case FAIL:
case ERROR:
return completed.get() ? s : Job.Status.RUNNING;
case IDLE:
case RUNNING:
default:
return s;
}
}

Expand All @@ -432,7 +456,7 @@ public void close() {
private void writeResult(JobResult result, File dest) throws IOException {
dest.getParentFile().mkdirs();
try (InputStream is = result.asStream();
OutputStream os = new FileOutputStream(dest)) {
OutputStream os = new FileOutputStream(dest)) {
byte buff[] = new byte[1024];
int read = 0;
while ((read = is.read(buff)) > 0) {
Expand Down

0 comments on commit 35c0c97

Please sign in to comment.