Skip to content

Commit

Permalink
Tools hive: LSF to Slurm migration (#842)
Browse files Browse the repository at this point in the history
  • Loading branch information
@azangru
azangru authored Jan 15, 2025
1 parent 1762c7d commit 5a7a664
Show file tree
Hide file tree
Showing 18 changed files with 96 additions and 80 deletions.
58 changes: 29 additions & 29 deletions tools_hive/conf/SiteDefs.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ package EnsEMBL::Tools_hive::SiteDefs;
### tools_hive plugin is a backend for running the ensembl tools jobs (BLAST, BLAT, VEP etc) using ensembl-hive.
### With this plugin, the jobs that are submitted by the website (via the tools plugin) are saved in the
### ENSEMBL_WEB_HIVE db and another process called 'beekeeper' (that could possibly be running on a different
### machine) continually looks at that database, runs the newly submitted jobs on LSF farm or on
### machine) continually looks at that database, runs the newly submitted jobs on the farm or on
### the local machine (LOCAL) where this process itself is running.

use strict;
Expand Down Expand Up @@ -52,7 +52,7 @@ sub update_conf {
'VR' => 'Hive',
}; # Overriding tools plugin variable
$SiteDefs::ENSEMBL_HIVE_HOSTS = []; # For LOCAL, the machine that runs the beekeeper unless it's same as the web server
# For LSF, list of hosts corresponding to the queues for all jobs plus the machine where
# For farm, list of hosts corresponding to the queues for all jobs plus the machine where
# beekeeper is running unless it's same as the web server
# Leave it blank if code is located on a shared disk to share between the web server and the machine(s)
# running beekeeper
Expand Down Expand Up @@ -81,30 +81,30 @@ sub update_conf {

# BLAST configs
$SiteDefs::ENSEMBL_BLAST_RUN_LOCAL = 1; # Flag if on, will run blast jobs on LOCAL meadow
$SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # LSF or LOCAL queue for blast jobs
$SiteDefs::ENSEMBL_BLAST_LSF_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on LSF
$SiteDefs::ENSEMBL_BLAST_QUEUE = 'highpri'; # farm or LOCAL queue for blast jobs
$SiteDefs::ENSEMBL_BLAST_FARM_TIMEOUT = undef; # Max timelimit a blast job is allowed to run on farm
$SiteDefs::ENSEMBL_BLAST_MEMORY_USAGE = 8; # Memory in GBs required for Blast jobs
$SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_BLAST_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blast queue (farm or LOCAL)
$SiteDefs::ENSEMBL_NCBIBLAST_BIN_PATH = '/path/to/ncbi-blast/bin'; # path to blast executables on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH = "/path/to/genes"; # path for the blast index files (other than DNA) on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_NCBIBLAST_DATA_PATH_DNA = "/path/to/blast/dna"; # path for the blast DNA index files on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_REPEATMASK_BIN_PATH = '/path/to/RepeatMasker'; # path to RepeatMasker executable on the farm host (or local machine if job running locally)

# BLAT configs
$SiteDefs::ENSEMBL_BLAT_RUN_LOCAL = 1; # Flag if on, will run blat jobs on LOCAL meadow
$SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # LSF or LOCAL queue for blat jobs
$SiteDefs::ENSEMBL_BLAT_LSF_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on LSF
$SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default LSF limit)
$SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on LSF node (or local machine if job running locally)
$SiteDefs::ENSEMBL_BLAT_QUEUE = 'highpri'; # farm or LOCAL queue for blat jobs
$SiteDefs::ENSEMBL_BLAT_FARM_TIMEOUT = undef; # Max timelimit a blat job is allowed to run on farm
$SiteDefs::ENSEMBL_BLAT_MEMORY_USAGE = undef; # Memory in GBs required for Blat jobs (undef for default farm limit)
$SiteDefs::ENSEMBL_BLAT_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the blat queue (farm or LOCAL)
$SiteDefs::ENSEMBL_BLAT_TWOBIT_DIR = "/path/to/blat/twobit"; # location where blat twobit files are located on farm node (or local machine if job running locally)
$SiteDefs::ENSEMBL_BLAT_QUERY_COMMAND = '/path/to/command [SPECIES].[ASSEMBLY]'; # optional command line that returns server:port for BLAT server for a given species and assembly

# VEP configs
$SiteDefs::ENSEMBL_VEP_RUN_LOCAL = 1; # Flag if on, will run VEP jobs on LOCAL meadow
$SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # LSF or LOCAL queue for VEP jobs
$SiteDefs::ENSEMBL_VEP_LSF_TIMEOUT = '3:00'; # Max timelimit a VEP job is allowed to run on LSF
$SiteDefs::ENSEMBL_VEP_QUEUE = 'highpri'; # farm or LOCAL queue for VEP jobs
$SiteDefs::ENSEMBL_VEP_FARM_TIMEOUT = '03:00:00'; # Max timelimit a VEP job is allowed to run on farm in HH::MM::SS
$SiteDefs::ENSEMBL_VEP_MEMORY_USAGE = 8; # Memory in GBs required for VEP jobs
$SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_VEP_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the VEP queue (farm or LOCAL)
$SiteDefs::ENSEMBL_VEP_CACHE_DIR = "/path/to/vep/cache"; # path to vep cache files
$SiteDefs::ENSEMBL_VEP_FASTA_DIR = "/path/to/fasta/files"; # path to bgzipped & indexed FASTA files for use by VEP
$SiteDefs::ENSEMBL_VEP_SCRIPT_DEFAULT_OPTIONS = { # Default options for command line vep script (keys with value undef get ignored)
Expand All @@ -115,21 +115,21 @@ sub update_conf {
'fork' => 4, # Enable forking, using 4 forks
};

$SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the LSF host (or local machine if job running locally)
$SiteDefs::ENSEMBL_VEP_PLUGIN_DATA_DIR = "/path/to/vep/plugin_data"; # path to vep plugin data files on the farm host (or local machine if job running locally)
$SiteDefs::ENSEMBL_VEP_PLUGIN_DIR = "VEP_plugins"; # path to vep plugin code (if does not start with '/', it's treated relative to ENSEMBL_HIVE_HOSTS_CODE_LOCATION)

push @{$SiteDefs::ENSEMBL_VEP_PLUGIN_CONFIG_FILES}, $SiteDefs::ENSEMBL_SERVERROOT.'/public-plugins/tools_hive/conf/vep_plugins_hive_config.txt';
# add extra hive specific configs required to run vep plugins
# LD configs
$SiteDefs::ENSEMBL_LD_RUN_LOCAL = 1; # Flag if on, will run LD jobs on LOCAL meadow
$SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # LSF or LOCAL queue for LD jobs
$SiteDefs::ENSEMBL_LD_LSF_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on LSF
$SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_LD_QUEUE = 'highpri'; # farm or LOCAL queue for LD jobs
$SiteDefs::ENSEMBL_LD_FARM_TIMEOUT = undef; # Max timelimit a LD job is allowed to run on farm
$SiteDefs::ENSEMBL_LD_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the LD queue (farm or LOCAL)

# Variant Recoder configs
$SiteDefs::ENSEMBL_VR_RUN_LOCAL = 1;
$SiteDefs::ENSEMBL_VR_QUEUE = 'highpri';
$SiteDefs::ENSEMBL_VR_LSF_TIMEOUT = undef;
$SiteDefs::ENSEMBL_VR_FARM_TIMEOUT = undef;
$SiteDefs::ENSEMBL_VR_ANALYSIS_CAPACITY = 500;
$SiteDefs::ENSEMBL_VR_SCRIPT_DEFAULT_OPTIONS = {
'host' => undef,
Expand All @@ -140,27 +140,27 @@ sub update_conf {

# Assembly Converter configs
$SiteDefs::ENSEMBL_AC_RUN_LOCAL = 1; # Flag if on, will run AC jobs on LOCAL meadow
$SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # LSF or LOCAL queue for AC jobs
$SiteDefs::ENSEMBL_AC_LSF_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on LSF
$SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_AC_QUEUE = 'highpri'; # farm or LOCAL queue for AC jobs
$SiteDefs::ENSEMBL_AC_FARM_TIMEOUT = undef; # Max timelimit an AC job is allowed to run on farm
$SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL)
$SiteDefs::ENSEMBL_CHAIN_FILE_DIR = '/path/to/assembly_converter/chain_files'; # path to chain files as required by assembly converter
$SiteDefs::ASSEMBLY_CONVERTER_BIN_PATH = '/path/to/CrossMap.py'; # path to CrossMap
$SiteDefs::WIGTOBIGWIG_BIN_PATH = '/path/to/wigToBigWig'; # path to wigToBigWig (required by CrossMap)
$SiteDefs::BIGWIGTOWIG_BIN_PATH = '/path/to/bigWigToWig'; # path to bigWigToWig (needed to RunnableDB/AssemblyConverter)

# ID History converter configs
$SiteDefs::ENSEMBL_IDM_RUN_LOCAL = 1; # Flag if on, will run ID mapper jobs on LOCAL meadow
$SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # LSF or LOCAL queue for ID mapper jobs
$SiteDefs::ENSEMBL_IDM_LSF_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on LSF
$SiteDefs::ENSEMBL_IDM_QUEUE = 'highpri'; # farm or LOCAL queue for ID mapper jobs
$SiteDefs::ENSEMBL_IDM_FARM_TIMEOUT = undef; # Max timelimit an ID mapper job is allowed to run on farm
$SiteDefs::ENSEMBL_IDM_MEMORY_USAGE = 6; # Memory in GBs required for IDMapper jobs
$SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (LSF or LOCAL)
$SiteDefs::ENSEMBL_IDM_ANALYSIS_CAPACITY = 500; # Number of jobs that can be run parallel in the queue (farm or LOCAL)
$SiteDefs::IDMAPPER_SCRIPT = 'ensembl-tools/scripts/id_history_converter/IDmapper.pl';
# Path to ID History converter script

# File Chameleon configs
$SiteDefs::ENSEMBL_FC_RUN_LOCAL = 1;
$SiteDefs::ENSEMBL_FC_QUEUE = 'highpri';
$SiteDefs::ENSEMBL_FC_LSF_TIMEOUT = undef;
$SiteDefs::ENSEMBL_FC_FARM_TIMEOUT = undef;
$SiteDefs::ENSEMBL_FC_ANALYSIS_CAPACITY = 500;

# Allele Frequency configs
Expand Down
4 changes: 2 additions & 2 deletions tools_hive/modules/EnsEMBL/Web/JobDispatcher/Hive.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ sub dispatch_job {
my $hive_dba = $self->_hive_dba;
my $job_adaptor = $self->_job_adaptor;

$self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name);
$self->{'_analysis'}{$logic_name} ||= $hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name);

# Submit job to hive db
my $hive_job = Bio::EnsEMBL::Hive::AnalysisJob->new(
Expand Down Expand Up @@ -72,7 +72,7 @@ sub delete_jobs {

# if (@hive_job_ids) {
# $self->_job_adaptor->remove_all(sprintf '`job_id` in (%s)', join(',', @hive_job_ids));
# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name_or_url($logic_name)->stats);
# $hive_dba->get_Queen->safe_synchronize_AnalysisStats($hive_dba->get_AnalysisAdaptor->fetch_by_logic_name($logic_name)->stats);
# }
}

Expand Down
2 changes: 1 addition & 1 deletion tools_hive/modules/EnsEMBL/Web/RunnableDB/VEP.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ sub run {
my $work_dir = $self->param('work_dir');
my $config = $self->param('config');
my $options = $self->param('script_options') || {};
my $log_file = "$work_dir/lsf_log.txt";
my $log_file = "$work_dir/farm_log.txt";

# path for VEP_plugins (gets pushed to INC by VEP::Runner)
if (my $plugins_path = $self->param('plugins_path')) {
Expand Down
23 changes: 12 additions & 11 deletions tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ use EnsEMBL::Web::Attributes;
sub logic_name :Abstract;
sub runnable :Abstract;
sub queue_name :Abstract;
sub is_lsf :Abstract;
sub lsf_timeout :Abstract;
sub is_farm :Abstract;
sub farm_timeout :Abstract;
sub memory_usage :Abstract;
sub analysis_capacity :Abstract;

Expand All @@ -52,7 +52,7 @@ sub pipeline_analyses {
'-parameters' => {},
'-rc_name' => $class->_resource_class_name,
'-analysis_capacity' => $class->analysis_capacity || 500,
'-meadow_type' => $class->is_lsf ? 'LSF' : 'LOCAL',
'-meadow_type' => $class->is_farm ? 'SLURM' : 'LOCAL',
'-max_retry_count' => 0,
'-failed_job_tolerance' => 100
}];
Expand All @@ -62,26 +62,27 @@ sub _format_resource_class {
## @private
my $class = shift;

return { 'LOCAL' => '' } unless $class->is_lsf;
return { 'LOCAL' => '' } unless $class->is_farm;

my $queue = $class->queue_name;
my $timeout = $class->lsf_timeout;
my $partition = $queue ? "--partition=$queue" : "";
my $timeout = $class->farm_timeout;
my $memory = $class->memory_usage;
my $default_timeout = '1-00:00:00'; # Days-HH::MM::SS
$timeout ||= $default_timeout;
$memory = $memory ? sprintf('%s', $memory * 1024) : '1600';

$timeout = $timeout ? " -W $timeout" : '';
$memory = $memory ? sprintf(' -M %s -R "rusage[mem=%1$s]"', $memory * 1024) : '';

return { 'LSF' => "-q $queue$timeout$memory" };
return { 'SLURM' => sprintf("$partition --time=%s --mem=%s%s -n 8 -N 1", $timeout, $memory, 'm') };
}

sub _resource_class_name {
## @private
my $class = shift;
my $rc = $class->_format_resource_class;
my $queue = $class->queue_name || '';
my $timeout = ($class->lsf_timeout || '') =~ s/\:.+$//r;
my $timeout = ($class->farm_timeout || '') =~ s/\:.+$//r;
my $memory = $class->memory_usage || '';
my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'W' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr;
my $str = sprintf('%s %s%s %s%s ', $queue, $timeout ? 'T' : '', $timeout, $memory ? 'M' : '', $memory) =~ s/\W+/-/gr;

return sprintf '%s%s', $str, substr(md5_hex(join(' ', %$rc)), 0, 4);
}
Expand Down
4 changes: 2 additions & 2 deletions tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AlleleFrequency.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig);
sub logic_name { 'AlleleFrequency' }
sub runnable { 'EnsEMBL::Web::RunnableDB::AlleleFrequency' }
sub queue_name { $SiteDefs::ENSEMBL_AF_QUEUE }
sub is_lsf { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL }
sub lsf_timeout { $SiteDefs::ENSEMBL_AF_LSF_TIMEOUT }
sub is_farm { !$SiteDefs::ENSEMBL_AF_RUN_LOCAL }
sub farm_timeout { $SiteDefs::ENSEMBL_AF_FARM_TIMEOUT }
sub memory_usage { $SiteDefs::ENSEMBL_AF_MEMORY_USAGE }
sub analysis_capacity { $SiteDefs::ENSEMBL_AF_ANALYSIS_CAPACITY }

Expand Down
4 changes: 2 additions & 2 deletions tools_hive/modules/EnsEMBL/Web/ToolsPipeConfig/AssemblyConverter.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ use parent qw(EnsEMBL::Web::ToolsPipeConfig);
sub logic_name { 'AssemblyConverter' }
sub runnable { 'EnsEMBL::Web::RunnableDB::AssemblyConverter' }
sub queue_name { $SiteDefs::ENSEMBL_AC_QUEUE }
sub is_lsf { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL }
sub lsf_timeout { $SiteDefs::ENSEMBL_AC_LSF_TIMEOUT }
sub is_farm { !$SiteDefs::ENSEMBL_AC_RUN_LOCAL }
sub farm_timeout { $SiteDefs::ENSEMBL_AC_FARM_TIMEOUT }
sub memory_usage { $SiteDefs::ENSEMBL_AC_MEMORY_USAGE }
sub analysis_capacity { $SiteDefs::ENSEMBL_AC_ANALYSIS_CAPACITY }

Expand Down
Loading

0 comments on commit 5a7a664

Please sign in to comment.