diff --git a/scripts/recreation_server/build_twitter_quadtree.sh b/scripts/recreation_server/build_twitter_quadtree.sh index eb7d28f0f6..3b96d4c9c5 100644 --- a/scripts/recreation_server/build_twitter_quadtree.sh +++ b/scripts/recreation_server/build_twitter_quadtree.sh @@ -2,15 +2,14 @@ # ----------------SLURM Parameters---------------- #SBATCH --partition normal,hns #SBATCH --ntasks 1 -#SBATCH --cpus-per-task=4 +#SBATCH --cpus-per-task=32 #SBATCH --mem-per-cpu=8000M #SBATCH --nodes 1 # Define how long the job will run d-hh:mm:ss -#SBATCH --time=6-00:00:00 -#SBATCH --qos=long +#SBATCH --time=48:00:00 # Get email notification when job finishes or fails #SBATCH --mail-user=dfisher5@stanford.edu -#SBATCH --mail-type=END,FAIL +#SBATCH --mail-type=END,FAIL,BEGIN #SBATCH -J build_quadtree #SBATCH -o build_quadtree # ----------------Load Modules-------------------- @@ -19,7 +18,8 @@ CONTAINER=ghcr.io/davemfish/invest:exp.rec-twitter TWEETS_DIR=/scratch/users/woodsp/invest/csv -find $TWEETS_DIR -name '*.csv' > ~/projects/rec-twitter/tweets_full_list.txt +TWEETS_LIST=$SCRATCH/tweets_full_list.txt +find $TWEETS_DIR -name '*.csv' > $TWEETS_LIST # invest repo already cloned into ~/invest cd ~/invest @@ -29,7 +29,7 @@ set -x # Be eXplicit about what's happening. FAILED=0 singularity run \ docker://$CONTAINER python scripts/recreation_server/build_twitter_quadtree.py \ - --csv_file_list=$HOME/projects/rec-twitter/tweets_full_list.txt \ + --csv_file_list=$TWEETS_LIST \ --workspace=$SCRATCH/twitter_quadtree \ --output_filename=global_twitter_qt.pickle \ - --n_cores=4 # match --cpus-per-task value + --n_cores=32 # match --cpus-per-task value