feat: Add LSF scripts and SLURM versions

.gitignore

@@ -1,6 +1,6 @@
 *.png
 *.csv
-*.sh
+# *.sh
 *.tsv
 *.txt
 *.log

harmonisationUtils/cron_scripts/refresh_hq.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/"
+
+bsub -oo "${logs}/refresh.out" -eo "${logs}/refresh.err" -q datamover -M4000 -R "rusage[mem=4000]" "cd /hps/software/users/parkinso/spot/gwas/prod/sw/harmonisation; cp hq.db hq.db.bak; source /hps/software/users/parkinso/spot/gwas/anaconda3/bin/activate; conda activate gwas-utils; queue-harmonisation --action refresh --source_dir /nfs/production/parkinso/spot/gwas/prod/data/summary_statistics --ftp_dir /nfs/ftp/public/databases/gwas/summary_statistics --harmonisation_dir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm"

harmonisationUtils/cron_scripts/refresh_hq_slurm.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/"
+
+sbatch --output="${logs}/refresh.out" --error="${logs}/refresh.err" --partition=datamover --mem=4000 --wrap="cd /hps/software/users/parkinso/spot/gwas/prod/sw/harmonisation; cp hq.db hq.db.bak; source /hps/software/users/parkinso/spot/gwas/anaconda3/bin/activate; conda activate gwas-utils; queue-harmonisation --action refresh --source_dir /nfs/production/parkinso/spot/gwas/prod/data/summary_statistics --ftp_dir /nfs/ftp/public/databases/gwas/summary_statistics --harmonisation_dir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm"

harmonisationUtils/cron_scripts/release_gwas_ssf.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/"
+day=$(date "+%Y%m%d") 
+
+bsub -q datamover -oo "${logs}/gwas-ssf.out" -eo "${logs}/gwas-ssf.err" -M4000 -R "rusage[mem=4000]" "cd /hps/software/users/parkinso/spot/gwas/prod/sw/harmonisation; source /hps/software/users/parkinso/spot/gwas/anaconda3/bin/activate; conda activate gwas-utils; mkdir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm/${day}; queue-harmonisation --action release --source_dir /nfs/production/parkinso/spot/gwas/prod/data/summary_statistics --ftp_dir /nfs/ftp/public/databases/gwas/summary_statistics --harmonisation_dir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm/${day} --number 50 --harmonisation_type v1"

harmonisationUtils/cron_scripts/release_gwas_ssf_slurm.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/"
+day=$(date "+%Y%m%d") 
+
+sbatch --output="${logs}/gwas-ssf.out" --error="${logs}/gwas-ssf.err" --partition=datamover --mem=4000 --wrap="cd /hps/software/users/parkinso/spot/gwas/prod/sw/harmonisation; source /hps/software/users/parkinso/spot/gwas/anaconda3/bin/activate; conda activate gwas-utils; mkdir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm/${day}; queue-harmonisation --action release --source_dir /nfs/production/parkinso/spot/gwas/prod/data/summary_statistics --ftp_dir /nfs/ftp/public/databases/gwas/summary_statistics --harmonisation_dir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm/${day} --number 50 --harmonisation_type v1"

harmonisationUtils/cron_scripts/release_pre_gwas_ssf.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/"
+day=$(date "+%Y%m%d")
+
+bsub -q datamover -oo "${logs}/pre-gwas-ssf.out" -eo "${logs}/pre-gwas-ssf.err" -M4000 -R "rusage[mem=4000]" "cd /hps/software/users/parkinso/spot/gwas/prod/sw/harmonisation; source /hps/software/users/parkinso/spot/gwas/anaconda3/bin/activate; conda activate gwas-utils; mkdir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/toharm/${day}; queue-harmonisation --action release --source_dir /nfs/production/parkinso/spot/gwas/prod/data/summary_statistics --ftp_dir /nfs/ftp/public/databases/gwas/summary_statistics --harmonisation_dir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/toharm/${day} --number 20 --harmonisation_type v0"

harmonisationUtils/cron_scripts/release_pre_gwas_ssf_slurm.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/"
+day=$(date "+%Y%m%d")
+
+sbatch --output="${logs}/pre-gwas-ssf.out" --error="${logs}/pre-gwas-ssf.err" --partition=datamover --mem=4000 --wrap="cd /hps/software/users/parkinso/spot/gwas/prod/sw/harmonisation; source /hps/software/users/parkinso/spot/gwas/anaconda3/bin/activate; conda activate gwas-utils; mkdir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/toharm/${day}; queue-harmonisation --action release --source_dir /nfs/production/parkinso/spot/gwas/prod/data/summary_statistics --ftp_dir /nfs/ftp/public/databases/gwas/summary_statistics --harmonisation_dir /hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/toharm/${day} --number 20 --harmonisation_type v0"

harmonisationUtils/cron_scripts/start_harmonisation.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+day=$(date -d "yesterday 13:00" "+%Y%m%d")
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/${day}"
+
+# DEV ENV
+#wrapper="/hps/software/users/parkinso/spot/gwas/dev/gwas-utils/harmonisationUtils/harmonisation_wrapper.sh"
+#version="nextflow"
+#pub_ftp="/nfs/ftp/private/gwas_cat/harm_test/success"
+
+wrapper="/hps/software/users/parkinso/spot/gwas/prod/scripts/gwas-utils/harmonisationUtils/harmonisation_wrapper.sh"
+ref="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/resources"
+to_harm="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm/${day}"
+pub_ftp="/nfs/ftp/public/databases/gwas/summary_statistics"
+failed="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/failed/${day}"
+version="v1.1.3"
+mail_add="gwas-dev-logs@ebi.ac.uk"
+file_type="gwas_ssf"
+nf_conf="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/container.config"
+
+cmd="${wrapper} ${ref} ${to_harm} ${pub_ftp} ${failed} ${version} ${logs} ${mail_add} ${file_type} ${nf_conf}"
+
+if [[ -d "$to_harm"  ]]; then
+	mkdir -p $logs
+	bsub -o "${logs}/nf.out" -e "${logs}/nf.err" -M8000 -R "rusage[mem=8000]" -g /gwas_ss_harmo "$cmd"
+else
+	echo "Nothing to harmonise today"
+fi

harmonisationUtils/cron_scripts/start_harmonisation_pre_standard.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+day=$(date -d "yesterday 13:00" "+%Y%m%d")
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/logs/${day}"
+
+wrapper="/hps/software/users/parkinso/spot/gwas/prod/scripts/gwas-utils/harmonisationUtils/harmonisation_wrapper.sh"
+ref="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/resources"
+to_harm="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/toharm/${day}"
+pub_ftp="/nfs/ftp/public/databases/gwas/summary_statistics"
+failed="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/failed/${day}"
+version="v1.0.4"
+mail_add="gwas-dev-logs@ebi.ac.uk"
+file_type="pre_gwas_ssf"
+nf_conf="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/container_pre_ssf.config"
+
+cmd="${wrapper} ${ref} ${to_harm} ${pub_ftp} ${failed} ${version} ${logs} ${mail_add} ${file_type} ${nf_conf}"
+
+if [[ -d "$to_harm"  ]]; then
+	mkdir -p $logs
+	bsub -o "${logs}/nf.out" -e "${logs}/nf.err" -M8000 -R "rusage[mem=8000]" -g /pre_gwas_ssf_harmo "$cmd"
+else
+	echo "Nothing to harmonise today"
+fi

harmonisationUtils/cron_scripts/start_harmonisation_pre_standard_slurm.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+day=$(date -d "yesterday 13:00" "+%Y%m%d")
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/logs/${day}"
+
+wrapper="/hps/software/users/parkinso/spot/gwas/prod/scripts/gwas-utils/harmonisationUtils/harmonisation_wrapper.sh"
+ref="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/resources"
+to_harm="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/toharm/${day}"
+pub_ftp="/nfs/ftp/public/databases/gwas/summary_statistics"
+failed="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/pre_gwas_ssf/failed/${day}"
+version="v1.0.4"
+mail_add="gwas-dev-logs@ebi.ac.uk"
+file_type="pre_gwas_ssf"
+nf_conf="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/container_pre_ssf.config"
+
+cmd="${wrapper} ${ref} ${to_harm} ${pub_ftp} ${failed} ${version} ${logs} ${mail_add} ${file_type} ${nf_conf}"
+
+if [[ -d "$to_harm"  ]]; then
+	mkdir -p $logs
+	sbatch --output="${logs}/nf.out" --error="${logs}/nf.err" --mem=8000 --time=02:00:00 --job-name=pre_gwas_ssf_harmo --wrap="${cmd}"
+else
+	echo "Nothing to harmonise today"
+fi

harmonisationUtils/cron_scripts/start_harmonisation_slurm.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+day=$(date -d "yesterday 13:00" "+%Y%m%d")
+
+logs="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/logs/${day}"
+
+# DEV ENV
+#wrapper="/hps/software/users/parkinso/spot/gwas/dev/gwas-utils/harmonisationUtils/harmonisation_wrapper.sh"
+#version="nextflow"
+#pub_ftp="/nfs/ftp/private/gwas_cat/harm_test/success"
+
+wrapper="/hps/software/users/parkinso/spot/gwas/prod/scripts/gwas-utils/harmonisationUtils/harmonisation_wrapper.sh"
+ref="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/resources"
+to_harm="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/toharm/${day}"
+pub_ftp="/nfs/ftp/public/databases/gwas/summary_statistics"
+failed="/hps/nobackup/parkinso/spot/gwas/data/sumstats/harmonisation/failed/${day}"
+version="v1.1.3"
+mail_add="gwas-dev-logs@ebi.ac.uk"
+file_type="gwas_ssf"
+nf_conf="/hps/software/users/parkinso/spot/gwas/prod/scripts/cron/container.config"
+
+cmd="${wrapper} ${ref} ${to_harm} ${pub_ftp} ${failed} ${version} ${logs} ${mail_add} ${file_type} ${nf_conf}"
+
+if [[ -d "$to_harm"  ]]; then
+	mkdir -p $logs
+	sbatch --output="${logs}/nf.out" --error="${logs}/nf.err" --mem=8000 --time=02:00:00 --job-name=gwas_ss_harmo --wrap="${cmd}"
+else
+	echo "Nothing to harmonise today"
+fi