Skip to content

Commit

Permalink
add option to run import metadata graphs using docker for testing
Browse files Browse the repository at this point in the history
  • Loading branch information
syphax-bouazzouni committed Jan 20, 2025
1 parent 9dcfc78 commit 5d1b424
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 25 deletions.
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,9 @@ logs/

# Ignore jEnv files
.java-version

processed_files/

queries.txt

graph_comparison.csv
16 changes: 8 additions & 8 deletions Gemfile.lock
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
GIT
remote: https://github.com/ontoportal-lirmm/goo.git
revision: f8ac7b00e8d8b46d1eea04de014175525c1cdd83
revision: 8d108c23a043039e9675b36f8f444d29a87b11fe
branch: development
specs:
goo (0.0.2)
Expand Down Expand Up @@ -29,7 +29,7 @@ GIT

GIT
remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
revision: e65d887616aaf4ae6f099437223d86515ffdca79
revision: 0aa6219c44143b94135e01c78eb94ad99a5e8b32
branch: development
specs:
ontologies_linked_data (0.0.1)
Expand All @@ -49,7 +49,7 @@ GIT

GIT
remote: https://github.com/ontoportal-lirmm/sparql-client.git
revision: 59251e59346c9a69a67c88552ba55a1244eec602
revision: 24bccbd0f4a5150fa6ce2af50d7c378c681027ea
branch: development
specs:
sparql-client (3.2.2)
Expand Down Expand Up @@ -101,7 +101,7 @@ GEM
capistrano (~> 3.1)
sshkit (~> 1.3)
coderay (1.1.3)
concurrent-ruby (1.3.4)
concurrent-ruby (1.3.5)
connection_pool (2.5.0)
cube-ruby (0.0.3)
dante (0.2.0)
Expand Down Expand Up @@ -138,7 +138,7 @@ GEM
google-cloud-errors (~> 1.0)
google-apis-analytics_v3 (0.16.0)
google-apis-core (>= 0.15.0, < 2.a)
google-apis-core (0.15.1)
google-apis-core (0.16.0)
addressable (~> 2.5, >= 2.5.1)
googleauth (~> 1.9)
httpclient (>= 2.8.3, < 3.a)
Expand All @@ -157,7 +157,7 @@ GEM
google-protobuf (>= 3.18, < 5.a)
googleapis-common-protos-types (~> 1.7)
grpc (~> 1.41)
googleapis-common-protos-types (1.17.0)
googleapis-common-protos-types (1.18.0)
google-protobuf (>= 3.18, < 5.a)
googleauth (1.11.2)
faraday (>= 1.0, < 3.a)
Expand Down Expand Up @@ -251,7 +251,7 @@ GEM
rexml (~> 3.2)
redis (5.3.0)
redis-client (>= 0.22.0)
redis-client (0.23.1)
redis-client (0.23.2)
connection_pool
representable (3.2.0)
declarative (< 0.1.0)
Expand Down Expand Up @@ -348,4 +348,4 @@ DEPENDENCIES
test-unit-minitest

BUNDLED WITH
2.3.14
2.3.3
6 changes: 6 additions & 0 deletions bin/migrations/compare_counts.rb
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,12 @@ def compare_graphs_with_files(graph_triples)
end
puts "MappingsCount count: #{count} with display=all in #{format("%.4f", time)}s"

count = 0
time = Benchmark.realtime do
count += LinkedData::Models::RestBackupMapping.where.all.count
end
puts "RestMappings count: #{count} with no display in #{format("%.4f", time)}s"

count = 0
time = Benchmark.realtime do
attr = LinkedData::Models::RestBackupMapping.attributes(:all) + LinkedData::Models::MappingProcess.attributes(:all)
Expand Down
9 changes: 9 additions & 0 deletions bin/migrations/import_metadata_graphs_to_store
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,15 @@ begin
# Optional profile to use for the import (vo: virtruoso, fs: 4store, gb: GraphDB)
profile = ARGV[1]

docker = ARGV[2] == "docker"

if docker
result = system("./start_ontoportal_services.sh #{profile}")
unless result
puts "Error starting services"
exit 1
end
end
# Check if processed_files directory exists
unless Dir.exist?(processed_dir)
puts "Processed files directory #{processed_dir} does not exist!"
Expand Down
36 changes: 19 additions & 17 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ services:
ports:
- "9393:9393"

mgrep-ut:
image: ontoportal/mgrep-ncbo:0.1
ports:
- "55556:55555"
# mgrep-ut:
# image: ontoportal/mgrep-ncbo:0.1
# ports:
# - "55556:55555"

redis-ut:
image: redis
Expand Down Expand Up @@ -117,20 +117,23 @@ services:
- ag

virtuoso-ut:
image: openlink/virtuoso-opensource-7:7.2
image: tenforce/virtuoso:virtuoso7.2.5
platform: linux/amd64
ports:
- 1111:1111
- 8890:8890
environment:
- DBA_PASSWORD=dba
- SPARQL_UPDATE=true
- VIRT_Parameters_NumberOfBuffers=2450000
- VIRT_Parameters_MaxDirtyBuffers=1837500
- VIRT_Parameters_NumOfThreads=100
- VIRT_Parameters_MaxMem=20000000000
- VIRT_Parameters_LogEnable=2
- VIRT_SPARQL_ResultSetMaxRows=1000000000000
- VIRT_SPARQL_MaxQueryExecutionTime=6000
- VIRT_SPARQL_MaxQueryCostEstimationTime=6000
profiles:
- vo
volumes:
- ./test/data/virtuoso.ini:/opt/virtuoso-opensource/database/virtuoso.ini
- ./test/data/virtuso_grant_write_permission.sh:/opt/virtuoso-opensource/initdb.d/virtuso_grant_write_permission.sh
entrypoint: >
bash -c "cd ..; ./bin/virtuoso-t -c ./database/virtuoso.ini ; ./initdb.d/virtuso_grant_write_permission.sh ; tail -f database/virtuoso.log"
ports:
- 1111:1111
- 8890:8890
healthcheck:
test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ]
start_period: 10s
Expand All @@ -150,11 +153,10 @@ services:
- 7200:7200
- 7300:7300
volumes:
- ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl
- ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/graphdb.ttl
- ./test/data/graphdb-test-load.nt:/opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt

entrypoint: >
bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb -Ddefault.min.distinct.threshold=3000 "
bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/graphdb.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb "
profiles:
- gb

Expand Down
85 changes: 85 additions & 0 deletions start_ontoportal_services.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/usr/bin/env bash
profile=$1
acronym=$2
set -e


if [ -z "$profile" ]; then
echo "Usage: $0 <acronym> <profile>"
exit 1
fi

BACKEND_TYPE=$profile
if [ "$BACKEND_TYPE" == "ag" ]; then
# AllegroGraph backend
export GOO_BACKEND_NAME="allegrograph"
export GOO_PORT="10035"
export GOO_PATH_QUERY="/repositories/ontoportal_test"
export GOO_PATH_DATA="/repositories/ontoportal_test/statements"
export GOO_PATH_UPDATE="/repositories/ontoportal_test/statements"
export COMPOSE_PROFILES="ag"

elif [ "$BACKEND_TYPE" == "fs" ]; then
# 4store backend
export GOO_PORT="9000"
export COMPOSE_PROFILES="fs"

elif [ "$BACKEND_TYPE" == "vo" ]; then
# Virtuoso backend
export GOO_BACKEND_NAME="virtuoso"
export GOO_PORT="8890"
export GOO_PATH_QUERY="/sparql"
export GOO_PATH_DATA="/sparql"
export GOO_PATH_UPDATE="/sparql"
export COMPOSE_PROFILES="vo"

elif [ "$BACKEND_TYPE" == "gb" ]; then
# Graphdb backend
export GOO_BACKEND_NAME="graphdb"
export GOO_PORT="7200"
export GOO_PATH_QUERY="/repositories/ontoportal"
export GOO_PATH_DATA="/repositories/ontoportal/statements"
export GOO_PATH_UPDATE="/repositories/ontoportal/statements"
else
echo "Error: Unknown backend type. Please set BACKEND_TYPE to 'ag', 'fs', or 'vo'."
fi

echo "###########################################################################"
echo "Stop and remove all containers, networks, and volumes and start fresh"
docker compose --profile fs --profile vo --profile gb --profile ag down --volumes --remove-orphans && docker compose --profile "$profile" up -d

echo "Waiting for all Docker services to start..."

while true; do
# Get the status of all containers
container_status=$(docker compose --profile "$profile" ps -a --format '{{.Names}} {{.State}}')

all_running=true
while read -r container state; do
if [ "$state" != "running" ] && [ "$state" != "exited" ]; then
all_running=false
break
fi
done <<< "$container_status"

# If all containers are running, exit the loop
if [ "$all_running" = true ]; then
echo "All containers are running!"
break
fi

# Wait before checking again
sleep 2
done

if [ -z "$acronym" ]; then
exit 0
fi

echo "###########################################################################"
echo "Create a new user and make it an admin"
bundle exec rake user:create[admin,[email protected],password]
bundle exec rake user:adminify[admin]
echo "###########################################################################"
echo "Create a new ontology $acronym and import it from a remote server"
bin/ncbo_ontology_import --admin-user admin -o "$acronym" --from https://data.stageportal.lirmm.fr --from-apikey 82602563-4750-41be-9654-36f46056a0db

0 comments on commit 5d1b424

Please sign in to comment.