config/script updates for codon cluster

Author: jamesamcl

configs/subgraph_configs/ebi_full_monarch.json

@@ -46,10 +46,6 @@
         "./configs/datasource_configs/otar.json",
         "./configs/datasource_configs/monarch.json",
         "./configs/datasource_configs/metabolights.json",
-        "./configs/datasource_configs/mondo_efo.json",
-        "./configs/datasource_configs/hett_pesticides_appril.json",
-        "./configs/datasource_configs/hett_pesticides_eu.json",
-        "./configs/datasource_configs/hett_pesticides_gb.json",
-        "./configs/datasource_configs/chembl.json"
+        "./configs/datasource_configs/mondo_efo.json"
     ]
 }

configs/subgraph_configs/hett.json

@@ -38,7 +38,6 @@
     "datasource_configs": [
         "./configs/datasource_configs/hett_pesticides_appril.json",
         "./configs/datasource_configs/hett_pesticides_eu.json",
-        "./configs/datasource_configs/hett_pesticides_gb.json",
-        "./configs/datasource_configs/chembl.json"
+        "./configs/datasource_configs/hett_pesticides_gb.json"
     ]
 }

nextflow/01_create_subgraph.nf

@@ -406,7 +406,7 @@ process create_solr_nodes_core {
 process create_solr_edges_core {
     cache "lenient"
     memory "64 GB" 
-    time "23h"
+    time "3d"
     cpus "16"
 
     publishDir "${params.tmp}/${params.config}/${params.subgraph}/solr_cores", overwrite: true, saveAs: { filename -> filename.replace("solr/data/", "") }

nextflow/codon_nextflow.config

@@ -1,5 +1,11 @@
 process.executor = 'slurm'
 
+process {
+    withName: build_equiv_groups {
+        memory = 300.GB
+    }
+}
+
 process {
     withName: materialise {
         memory = 96.GB
@@ -55,6 +61,7 @@ process {
 process {
     withName: create_neo {
         cpus = 32
+	memory = 72.GB
     }
 }
 

scripts/dataload_codon.py

@@ -12,7 +12,12 @@
 for subgraph in config['subgraphs']:
     print(f"===== LOADING SUBGRAPH: {subgraph} =====")
     os.environ['GREBI_SUBGRAPH'] = subgraph
-    os.system(f'nextflow {GREBI_HOME}/nextflow/01_create_subgraph.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
+    res = os.system(f'nextflow {GREBI_HOME}/nextflow/01_create_subgraph.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
+    if res != 0:
+        exit(res)
     print(f"===== FINISHED LOADING SUBGRAPH: {subgraph} =====")
 
-os.system(f'nextflow {GREBI_HOME}/nextflow/02_create_dbs.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
+res = os.system(f'nextflow {GREBI_HOME}/nextflow/02_create_dbs.nf -c {GREBI_HOME}/nextflow/codon_nextflow.config')
+if res != 0:
+    exit(res)
+

scripts/dataload_codon.sh

@@ -4,12 +4,14 @@ export GREBI_TMP=/hps/nobackup/parkinso/spot/grebi/tmp
 export GREBI_CONFIG=ebi
 export GREBI_IS_EBI=true
 export GREBI_TIMESTAMP=$(date +%Y_%m_%d__%H_%M)
+export GREBI_MAX_ENTITIES=1000000000
 module load nextflow-22.10.1-gcc-11.2.0-ju5saqw
 module load python
 export PYTHONPATH="/homes/spotbot/.local/lib/python3.6/site-packages:$PYTHONPATH"
 cd /hps/nobackup/parkinso/spot/grebi/
 export PYTHONUNBUFFERED=true
 srun -p datamover --time 1:0:0 --mem 8g bash -c "rm -rf /nfs/public/rw/ontoapps/grebi/staging && mkdir /nfs/public/rw/ontoapps/grebi/staging"
-srun --time 23:0:0 --mem 8g bash -c "rm -rf work tmp && python3 ${GREBI_HOME}/scripts/dataload_codon.py"
+srun --time 3-0:0:0 --mem 8g bash -c "rm -rf work tmp && python3 ${GREBI_HOME}/scripts/dataload_codon.py"
+#srun --time 23:0:0 --mem 8g bash -c "python3 ${GREBI_HOME}/scripts/dataload_codon.py"