Skip to content

Commit

Permalink
ExportETL development. For the Malaria Plant DB case, works until the…
Browse files Browse the repository at this point in the history 
… InVivoPharmaco sheet
  • Loading branch information
@acheype
acheype committed Jul 7, 2015
1 parent 4782aaf commit eeb6268
Show file tree
Hide file tree
Showing 22 changed files with 484 additions and 193 deletions.
144 changes: 144 additions & 0 deletions malariaplantdb.log
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion src/main/java/nc/ird/malariaplantdb/entities/InVitroPharmaco.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

/**
* In vitro pharmacology entity
*
* <p>
* Represents for the plant ingredients of a publication the relevant data in in vitro pharmacology
*
* @author acheype
Expand Down Expand Up @@ -96,6 +96,7 @@ public class InVitroPharmaco {
@Column(name = "measure_method")
private String measureMethod;

// TODO put a BigDecimal for the float properties in each entity
@JsonView(View.Detailed.class)
private Float concentration;

Expand Down
51 changes: 28 additions & 23 deletions src/main/java/nc/ird/malariaplantdb/service/rest/ImportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import lombok.extern.slf4j.Slf4j;
import nc.ird.malariaplantdb.entities.*;
import nc.ird.malariaplantdb.entities.Compiler;
import nc.ird.malariaplantdb.repositories.EthnologyRepo;
import nc.ird.malariaplantdb.repositories.PlantIngredientRepo;
import nc.ird.malariaplantdb.repositories.PublicationRepo;
import nc.ird.malariaplantdb.repositories.SpeciesRepo;
Expand All @@ -11,6 +12,8 @@
import nc.ird.malariaplantdb.service.xls.ImportStatus;
import nc.ird.malariaplantdb.service.xls.dto.PlantIngredients;
import nc.ird.malariaplantdb.service.xls.exceptions.ImportException;
import nc.ird.malariaplantdb.service.xls.exceptions.ImportRuntimeException;
import org.apache.commons.beanutils.PropertyUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

Expand All @@ -21,6 +24,7 @@
import javax.ws.rs.core.MediaType;
import java.io.BufferedInputStream;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.List;

/**
Expand All @@ -35,6 +39,10 @@
@Slf4j
public class ImportService {

static final private String[] PLANT_INGREDIENTS_PROPERTIES = {"plantIngredient1", "plantIngredient2", "plantIngredient3",
"plantIngredient4", "plantIngredient5", "plantIngredient6", "plantIngredient7", "plantIngredient8",
"plantIngredient9", "plantIngredient10"};

@Inject
private PublicationRepo publiRepo;

Expand All @@ -44,6 +52,9 @@ public class ImportService {
@Inject
private PlantIngredientRepo plantIngredientRepo;

@Inject
private EthnologyRepo ethnologyRepo;

@Transactional
@Produces(MediaType.APPLICATION_JSON)
@POST
Expand Down Expand Up @@ -78,34 +89,28 @@ private void persistEntities(ClassMap entities) {
}

List<Species> species = entities.getList(Species.class);
for (Species sp : species) {
speciesRepo.save(sp);
}
species.forEach(speciesRepo::save);

persistBundleOfPlantsIngredients(entities.getList(PlantIngredients.class));

List<Ethnology> ethnoNotes = entities.getList(Ethnology.class);
ethnoNotes.forEach(ethnologyRepo::save);
}

private void persistBundleOfPlantsIngredients(List<PlantIngredients> bundleOfPlantIngredients) {
for (PlantIngredients plantIngredients : bundleOfPlantIngredients) {
persistPlantIngredient(plantIngredients.getSpecies1(), plantIngredients.getPartUsed1());
persistPlantIngredient(plantIngredients.getSpecies2(), plantIngredients.getPartUsed2());
persistPlantIngredient(plantIngredients.getSpecies3(), plantIngredients.getPartUsed3());
persistPlantIngredient(plantIngredients.getSpecies4(), plantIngredients.getPartUsed4());
persistPlantIngredient(plantIngredients.getSpecies5(), plantIngredients.getPartUsed5());
persistPlantIngredient(plantIngredients.getSpecies6(), plantIngredients.getPartUsed6());
persistPlantIngredient(plantIngredients.getSpecies7(), plantIngredients.getPartUsed7());
persistPlantIngredient(plantIngredients.getSpecies8(), plantIngredients.getPartUsed8());
persistPlantIngredient(plantIngredients.getSpecies9(), plantIngredients.getPartUsed9());
persistPlantIngredient(plantIngredients.getSpecies10(), plantIngredients.getPartUsed10());
}
}

private void persistPlantIngredient(Species species, String partUsed) {
if (species != null && partUsed != null) {
PlantIngredient plantIngredient = new PlantIngredient();
plantIngredient.setSpecies(species);
plantIngredient.setPartUsed(partUsed);

plantIngredientRepo.save(plantIngredient);
for (int i = 0; i < 10; i++) {
try {
PlantIngredient plantIngredient = (PlantIngredient) PropertyUtils.getProperty(plantIngredients,
PLANT_INGREDIENTS_PROPERTIES[i]);
if (plantIngredient.getSpecies() != null && plantIngredient.getPartUsed() != null)
plantIngredientRepo.save(plantIngredient);
} catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException |
IllegalArgumentException e) {
throw new ImportRuntimeException("An unexpected error occurs during saving the entities " +
"in the database", e);
}
}
}
}

Expand Down
5 changes: 4 additions & 1 deletion src/main/java/nc/ird/malariaplantdb/service/xls/ExcelETL.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,22 +77,25 @@ private void readXlsFile(InputStream xlsDataInputStream) throws ImportException
ExcelReader.ReaderResult readerResult = excelReader.read(xlsDataInputStream);
getImportStatus().getReadErrors().addAll(readerResult.getCellErrors());
dtosMap = readerResult.getDtosMap();
// TODO need to sort the read errors
}

private void checkDtos() {
ExcelChecker excelChecker = new ExcelChecker(getSheetInfos());
getImportStatus().getBusinessErrors().addAll(excelChecker.checkBusinessRules(dtosMap));
// TODO sort the business errors
}

private void loadDtos() {
ExcelLoader excelLoader = new ExcelLoader(getSheetInfos());
ExcelLoader.LoaderResult loaderResult = excelLoader.loadEntities(dtosMap);
getImportStatus().getIntegrityErrors().addAll(loaderResult.getCellErrors());
// TODO sort the integrity errors
//getImportStatus().getIntegrityErrors().stream().sorted()
entitiesMap = loaderResult.getEntitiesMap();
}

private List<SheetInfo> buildSheetInfos() {

Reflections reflections = new Reflections(dtosBasePackage, new SubTypesScanner(false),
new TypeAnnotationsScanner());
Set<Class<?>> dtoClasses = reflections.getTypesAnnotatedWith(ImportDto.class);
Expand Down
48 changes: 26 additions & 22 deletions src/main/java/nc/ird/malariaplantdb/service/xls/ExcelLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,8 @@ private void fillRefEntities(ClassMap dtosMap, LoaderResult loaderResult) {
handleXlsEntityNotFoundException(loaderResult, sheetInfo, i, refInfo, entityRefType,
identifierValues, xlsEntityNotFoundException);
} catch (DbEntityNotFoundException dbEntityNotFoundException) {
handleDbEntityNotFoundException(loaderResult, sheetInfo, i, refInfo, entityRefType,
identifierValues, dbEntityNotFoundException);
handleDbEntityNotFoundException(loaderResult, sheetInfo, i, refInfo, identifierValues,
dbEntityNotFoundException);
}
}
} catch (InstantiationException | IllegalAccessException | InvocationTargetException |
Expand All @@ -199,35 +199,38 @@ private void handleXlsEntityNotFoundException(LoaderResult loaderResult, SheetIn
assert (refSheetInfo != null);

StringBuilder valuesMap = new StringBuilder();
assert (refInfo.getRefIdentifierProperties().length == identifierValues.size());
for (int j = 0; j < refInfo.getRefIdentifierProperties().length; j++) {
String columnLabel = refSheetInfo.getColumnInfoByOutputProperty(refInfo
.getRefIdentifierProperties()[j]).getColumnLabel();
valuesMap.append(columnLabel).append(" = '").append
(identifierValues.get(j)).append("'");
if (j != refInfo.getRefIdentifierProperties().length - 1)
// refInfo.getRefIdentifierProperties().length != identifierValues.size() with extreme cases fillers
for (int j = 0; j < identifierValues.size(); j++) {
ColumnInfo columnInfo = refSheetInfo.getColumnInfoByOutputProperty(refInfo
.getRefIdentifierProperties()[j]);
if (columnInfo != null)
valuesMap.append(columnInfo.getColumnLabel()).append(" = '")
.append(identifierValues.get(j)).append("'");
else
valuesMap.append("'").append(identifierValues.get(j)).append("'");
if (j != identifierValues.size() - 1)
valuesMap.append(", ");
}

loaderResult.getCellErrors().add(
new CellError(String.format("Can't find in the '%s' sheet a line with %s " +
"value%s : %s",
refSheetInfo.getSheetLabel(),
identifierValues.size() > 1 ? "these" : "this",
identifierValues.size() > 1 ? "s" : "",
valuesMap
),
new CellError(
String.format("Can't find in the '%s' sheet a line with %s value%s : %s",
refSheetInfo.getSheetLabel(),
identifierValues.size() > 1 ? "these" : "this",
identifierValues.size() > 1 ? "s" : "",
valuesMap),
sheetInfo.getSheetLabel(),
sheetInfo.getStartRow() + i,
Arrays.stream(refInfo.getDtoIdentifierProperties())
.map(dtoProp -> sheetInfo.getColumnInfoByDtoProperty(dtoProp).getColumnLabel())
.collect(Collectors.joining(", ")),
xlsEntityNotFoundException
)
);
}

private void handleDbEntityNotFoundException(LoaderResult loaderResult, SheetInfo sheetInfo, int i, EntityRefInfo
refInfo, Class entityRefType, ArrayList<Object> identifierValues, DbEntityNotFoundException
refInfo, ArrayList<Object> identifierValues, DbEntityNotFoundException
xlsEntityNotFoundException) {

StringBuilder valuesMap = new StringBuilder();
Expand All @@ -242,14 +245,15 @@ private void handleDbEntityNotFoundException(LoaderResult loaderResult, SheetInf
}

loaderResult.getCellErrors().add(
new CellError(String.format("Can't find in the database an entity with %s value%s : %s",
identifierValues.size() > 1 ? "these" : "this",
identifierValues.size() > 1 ? "s" : "",
valuesMap
),
new CellError(
String.format("Can't find in the database an entity with %s value%s : %s",
identifierValues.size() > 1 ? "these" : "this",
identifierValues.size() > 1 ? "s" : "",
valuesMap),
sheetInfo.getSheetLabel(),
sheetInfo.getStartRow() + i,
Arrays.stream(refInfo.getDtoIdentifierProperties())
.map(dtoProp -> sheetInfo.getColumnInfoByDtoProperty(dtoProp).getColumnLabel())
.collect(Collectors.joining(", ")),
xlsEntityNotFoundException
)
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/nc/ird/malariaplantdb/service/xls/annotations/EntityRef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

/**
* @return the entity ref class used to do the comparison. Usually this class is not specified,
* and inferred from the outputProperty
* and inferred from the outputProperty. May be useful per example if the outputProperty refer to a list.
*/
public Class entityRefType() default NULL_TYPE.class;

Expand Down
42 changes: 17 additions & 25 deletions src/main/java/nc/ird/malariaplantdb/service/xls/dto/EthnologyLine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,48 +32,36 @@
@EntityRef(
dtoIdentifierProperties = {"plantIngredients"},
dtoIdentifierTransformer = TrimStringTransformer.class,
entityRefIdentifierProperties = {"plantIngredient1", "plantIngredient2", "plantIngredient3",
"plantIngredient4", "plantIngredient5", "plantIngredient6", "plantIngredient7", "plantIngredient8",
"plantIngredient9", "plantIngredient10"},
entityRefType = PlantIngredients.class,
filler = PlantIngredientsXlsEntityRefFiller.class,
outputProperty = "plantIngredients"
),
@EntityRef(
dtoIdentifierProperties = {"ethnoRelevancyRef"},
dtoIdentifierTransformer = TrimStringTransformer.class,
entityRefIdentifierProperties = {"title"},
filler = XlsEntityRefFiller.class,
outputProperty = "ethnoRelevancyRef"
)
})
@EmptyOrNotIfPropertyValue.List({
@EmptyOrNotIfPropertyValue(
message = "As 'Traditional recipe' is YES, 'Traditional recipe details' must not be empty",
criteriaProperty = "isTraditionalRecipe",
criteriaValues = {"true"},
testedProperty = "traditionalRecipeDetails",
isEmpty = false
),
@EmptyOrNotIfPropertyValue(
message = "As 'Traditional recipe' is NO, 'Traditional recipe details' must be empty",
criteriaProperty = "isTraditionalRecipe",
criteriaValues = {"false"},
testedProperty = "traditionalRecipeDetails",
isEmpty = true
),
@EmptyOrNotIfPropertyValue(
message = "As 'Traditional recipe' is YES, 'Preparation mode in traditional recipe' must not be empty",
criteriaProperty = "isTraditionalRecipe",
criteriaValues = {"true"},
testedProperty = "preparationMode",
isEmpty = false
),
@EmptyOrNotIfPropertyValue(
message = "As 'Traditional recipe' is NO, 'Preparation mode in traditional recipe' must be empty",
criteriaProperty = "isTraditionalRecipe",
criteriaValues = {"false"},
testedProperty = "preparationMode",
isEmpty = true
),
@EmptyOrNotIfPropertyValue(
message = "As 'Traditional recipe' is YES, 'Administration route in traditional recipe' must not be " +
"empty",
criteriaProperty = "isTraditionalRecipe",
criteriaValues = {"true"},
testedProperty = "administrationRoute",
isEmpty = false
),
@EmptyOrNotIfPropertyValue(
message = "As 'Traditional recipe' is NO, 'Administration route in traditional recipe' must be empty",
criteriaProperty = "isTraditionalRecipe",
Expand All @@ -90,16 +78,20 @@ public class EthnologyLine {

@ImportProperty(columnLetterRef = "B", columnLabel = "Plant ingredient(s) used")
@NotEmpty(message = "The cell is empty or the value invalid")
// TODO reactivate the validator just below
// @Pattern(regexp = "^(([a-zA-ZÀ-ÿ &\\.\\-\\(\\)]+),([a-zA-ZÀ-ÿ \\-\\.]+)/)*" +
// "([a-zA-ZÀ-ÿ &\\.\\-\\(\\)]]+),([a-zA-ZÀ-ÿ \\-\\.]+)$",
// message = "The plant ingredient(s) value is not well formatted. Please enter each species name first, " +
// "a coma (,) then the part used. For several plant ingredients, please separate each plant ingredient " +
// "by a slash (/).")
private String plantIngredients;

@ImportProperty(columnLetterRef = "C", columnLabel = "Ethnopharmalogical relevancy",
propertyLoader = @PropertyLoader(outputProperty = "ethnoRelevancy",
transformer = TrimStringTransformer.class))
private String ethnoRelevancy;

@ImportProperty(columnLetterRef = "D", columnLabel = "Reference for ethnopharmalogical relevancy",
propertyLoader = @PropertyLoader(outputProperty = "ethnoRelevancyRef",
transformer = TrimStringTransformer.class))
@ImportProperty(columnLetterRef = "D", columnLabel = "Reference for ethnopharmalogical relevancy")
private String ethnoRelevancyRef;

@ImportProperty(columnLetterRef = "E", columnLabel = "Treatment type",
Expand Down
Loading

0 comments on commit eeb6268

Please sign in to comment.