minimal aggregation of value sets

notebooks/environmental_context_value_sets/.gitignore

@@ -1 +1,3 @@
 env-context-voting-sheets-29d0d970ca1d.json
+envo.owl
+

project.Makefile

@@ -319,4 +319,169 @@ notebooks/environmental_context_value_sets/nmdc_env_context_subset_membership.ts
 	$(RUN) python src/nmdc_submission_schema/scripts/create_env_context_robot_template.py \
 		--envo-owl-path notebooks/environmental_context_value_sets/envo.owl \
 		--schema-path $< \
-		--output-file $@
+		--output-file $@
+
+local/aggregate_value_sets.tsv:
+	$(RUN) python src/nmdc_submission_schema/scripts/aggregate_value_sets.py \
+		--output $@ \
+		--file notebooks/environmental_context_value_sets/plant_associated/env_broad_scale/post_google_sheets_plant_associated_env_broad_scale.tsv \
+		--file notebooks/environmental_context_value_sets/plant_associated/env_local_scale/post_google_sheets_plant_associated_env_local_scale.tsv \
+		--file notebooks/environmental_context_value_sets/plant_associated/env_medium/post_google_sheets_plant_associated_env_medium.tsv \
+		--file notebooks/environmental_context_value_sets/sediment/env_broad_scale/post_google_sheets_sediment_env_broad_scale.tsv \
+		--file notebooks/environmental_context_value_sets/sediment/env_local_scale/post_google_sheets_sediment_env_local_scale.tsv \
+		--file notebooks/environmental_context_value_sets/sediment/env_medium/post_google_sheets_sediment_env_medium.tsv \
+		--file notebooks/environmental_context_value_sets/soil/env_broad_scale/post_google_sheets_soil_env_broad_scale.tsv \
+		--file notebooks/environmental_context_value_sets/soil/env_local_scale/post_google_sheets_soil_env_local_scale.tsv \
+		--file notebooks/environmental_context_value_sets/soil/env_medium/post_google_sheets_soil_env_medium.tsv \
+		--file notebooks/environmental_context_value_sets/water/env_broad_scale/post_google_sheets_water_env_broad_scale.tsv \
+		--file notebooks/environmental_context_value_sets/water/env_local_scale/post_google_sheets_water_env_local_scale.tsv \
+		--file notebooks/environmental_context_value_sets/water/env_medium/post_google_sheets_water_env_medium.tsv
+
+local/soil_env_broad_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Soil	env_broad_scale' $< | cut -f 3 > $@
+
+local/soil_env_local_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Soil	env_local_scale' $< | cut -f 3 > $@
+
+local/soil_env_medium_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Soil	env_medium' $< | cut -f 3 > $@
+
+local/water_env_broad_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Water	env_broad_scale' $< | cut -f 3 > $@
+
+local/water_env_local_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Water	env_local_scale' $< | cut -f 3 > $@
+
+local/water_env_medium_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Water	env_medium' $< | cut -f 3 > $@
+
+local/plant_associated_env_broad_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Plant	env_broad_scale' $< | cut -f 3 > $@
+
+local/plant_associated_env_local_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Plant	env_local_scale' $< | cut -f 3 > $@
+
+local/plant_associated_env_medium_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Plant	env_medium' $< | cut -f 3 > $@
+
+local/sediment_env_broad_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Sediment	env_broad_scale' $< | cut -f 3 > $@
+
+local/sediment_env_local_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Sediment	env_local_scale' $< | cut -f 3 > $@
+
+local/sediment_env_medium_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'Sediment	env_medium' $< | cut -f 3 > $@
+
+local/env_broad_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'env_broad_scale' $< | cut -f 3 | sort | uniq > $@
+
+local/env_local_scale_curies.txt: local/aggregate_value_sets.tsv
+	egrep 'env_local_scale' $< | cut -f 3 | sort | uniq > $@
+
+
+local/soil_env_broad_scale_enrichment.tsv: local/soil_env_broad_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/soil_env_local_scale_enrichment.tsv: local/soil_env_local_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/soil_env_medium_enrichment.tsv: local/soil_env_medium_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/water_env_broad_scale_enrichment.tsv: local/water_env_broad_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/water_env_local_scale_enrichment.tsv: local/water_env_local_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/water_env_medium_enrichment.tsv: local/water_env_medium_curies.txt
+	# --filter-redundant limits this to liquid water
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/plant_associated_env_broad_scale_enrichment.tsv: local/plant_associated_env_broad_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/plant_associated_env_local_scale_enrichment.tsv: local/plant_associated_env_local_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/plant_associated_env_medium_enrichment_by_envo.tsv: local/plant_associated_env_medium_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/plant_associated_env_medium_enrichment_by_po.tsv: local/plant_associated_env_medium_curies.txt
+	$(RUN) runoak -i sqlite:obo:po enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/sediment_env_medium_enrichment.tsv: local/sediment_env_medium_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/sediment_env_broad_scale_enrichment.tsv: local/sediment_env_broad_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/sediment_env_local_scale_enrichment.tsv: local/sediment_env_local_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/env_broad_scale_enrichment.tsv: local/env_broad_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<
+
+local/env_local_scale_enrichment.tsv: local/env_local_scale_curies.txt
+	$(RUN) runoak -i sqlite:obo:envo enrichment \
+		--ontology-only \
+		-p i \
+		-O tsv \
+		-o $@ .idfile $<

src/nmdc_submission_schema/scripts/aggregate_value_sets.py

@@ -0,0 +1,56 @@
+import os
+import pandas as pd
+import click
+
+
+@click.command()
+@click.option(
+    '--file', '-f',
+    type=click.Path(exists=True),
+    multiple=True,
+    help="Paths to the input TSV files. This option can be used multiple times to specify multiple files."
+)
+@click.option(
+    '--output', '-o',
+    type=click.Path(),
+    default="Combined_Environmental_Context_Data.tsv",
+    help="Path to the output TSV file. Defaults to 'Combined_Environmental_Context_Data.tsv'."
+)
+def combine_context_files(file, output):
+    """Combine environmental context files into a single TSV file with Extension, Context Field, ENVO Class CURIE, and Label columns."""
+    combined_data = []
+
+    for file_path in file:
+        # Parse extension and context field from the filename
+        filename = os.path.basename(file_path)
+        parts = filename.split("_")
+        extension = parts[3].capitalize()  # e.g., "soil" or "water"
+
+        # Correctly map context fields to their full names
+        if "broad" in filename:
+            context_field = "env_broad_scale"
+        elif "local" in filename:
+            context_field = "env_local_scale"
+        elif "medium" in filename:
+            context_field = "env_medium"
+        else:
+            raise ValueError(f"Could not determine context field from filename: {filename}")
+
+        # Load the file into a DataFrame
+        df = pd.read_csv(file_path, sep="\t")
+
+        # Extract CURIEs and labels from the "id" and "label" columns
+        if "id" in df.columns and "label" in df.columns:
+            for curie, label in zip(df["id"].dropna(), df["label"].dropna()):
+                combined_data.append((extension, context_field, curie, label))
+
+    # Convert the combined data into a DataFrame
+    final_df = pd.DataFrame(combined_data, columns=["extension", "env_context_field", "class_curie", "label"])
+
+    # Save the DataFrame to a TSV file
+    final_df.to_csv(output, sep="\t", index=False)
+    click.echo(f"Combined data saved to: {output}")
+
+
+if __name__ == "__main__":
+    combine_context_files()