Skip to content

Commit

Permalink
Use the same perl interpreter for all jobs; find bin, perl and java p…
Browse files Browse the repository at this point in the history 
…aths consistently. Issue #2215.
  • Loading branch information
@brianwalenz
brianwalenz committed Jul 21, 2024
1 parent 6eb6d2c commit 86ad90d
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 68 deletions.
25 changes: 17 additions & 8 deletions src/pipelines/canu/Defaults.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,9 @@ sub setDefaults () {
$global{"version"} = undef; # Set in setVersion() once we know where binaries are.
$global{"availablehosts"} = undef; # Internal list of cpus-memory-nodes describing the grid.

$global{"perl"} = $^X; # Use whatever perl interpreter is in use now for resubmission.
$global{"perl"} = `command -v perl` if (!defined($global{"perl"}));

$global{"localmemory"} = 0; # Amount of memory on the local host, set in Grid_Local.pm
$global{"localthreads"} = 0;

Expand All @@ -809,7 +812,7 @@ sub setDefaults () {

setDefault("showNext", undef, "Don't run any commands, just report what would run");
setDefault("shell", "/bin/sh", "Command interpreter to use; sh-compatible (e.g., bash), NOT C-shell (csh or tcsh); default '/bin/sh'");

setDefault("minimap", "minimap2", "Path to minimap2; default 'minimap2'");

setDefault("java", $java, "Java interpreter to use; at least version 1.8; default 'java'");
Expand Down Expand Up @@ -1055,22 +1058,28 @@ sub setVersion ($) {


sub checkJava () {
my $java;

return if ((getGlobal("corOverlapper") ne "mhap") &&
(getGlobal("obtOverlapper") ne "mhap") &&
(getGlobal("utgOverlapper") ne "mhap"));

my $java = getGlobal("java");
my $javaUse64Bit = getGlobal("javaUse64Bit");
my $versionStr = "unknown";
my $version = 0;
my @javaVersionStrings;
$java = getGlobal("java");
$java = `command -v $java`; # See Execution.pm getBinDirectoryShellCode()
$java =~ s/^\s+//;
$java =~ s/\s+$//;

if ($java =~ m/^\./) {
addCommandLineError("ERROR: path to java '$java' must not be a relative path.\n");
}

# We've seen errors running just this tiny java if too many copies are ran at the same time.
# So, run it twice, if needed, with a little random delay between.
# We've seen errors running just this tiny java if too many copies are
# run at the same time. So, run it twice, if needed, with a little
# random delay between.

my @javaVersionStrings;
my $versionStr = "unknown";
my $version = 0;

for (my $iter=0; $iter<2; $iter++) {
open(F, "$java -Xmx1g -showversion 2>&1 |");
Expand Down
85 changes: 28 additions & 57 deletions src/pipelines/canu/Execution.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,27 +297,6 @@ sub resetIteration ($) {



# Decide what bin directory to use.
#
# When we are running on the grid, the path of this perl script is NOT always the correct
# architecture. If the submission host is FreeBSD, but the grid is Linux, the BSD box will submit
# FreeBSD/bin/canu to the grid. Unless it knows which grid host it will run on in advance, there
# is no way to pick the correct one. The grid host then has to have enough smarts to choose the
# correct binaries, and that is what we're doing here.
#
# To make it more trouble, shell scripts need to do all this by themselves.
#
#sub getInstallDirectory () {
# my $installDir = ;
#
# if ($installDir =~ m!^(.*)/\w+-\w+/bin$!) {
# $installDir = $1;
# }
#
# return($installDir);
#}


# Emits a block of shell code to parse the grid task id and offset.
# Expects zero or one argument, which is interpreted different in grid and non-grid mode.
# Off grid - the job to run
Expand Down Expand Up @@ -401,59 +380,52 @@ sub getLimitShellCode () {
#
sub getBinDirectory () {
return($FindBin::RealBin);

#my $idir = getInstallDirectory();
#my $path = $idir;
#
#$path = "$idir/bin" if (-d "$idir/bin");
#
#return($path);
}


# Emits a block of shell code to locate binaries during shell scripts. See comments on
# getBinDirectory.
#
sub getBinDirectoryShellCode () {
my $idir = $FindBin::RealBin;
my $string;

# First, run any preExec command that might exist.

if (defined(getGlobal("preExec"))) {
$string .= "# Pre-execution commands.\n";
$string .= "\n";
$string .= getGlobal('preExec') . "\n";
$string .= "\n";
}

# Then, setup and report paths.

my $javaPath = getGlobal("java");
my $canu = "\$bin/" . basename($0); # NOTE: $bin decided at script run time

$string = "\n";
$string .= "# Pre-execution commands.\n" if (defined(getGlobal("preExec")));
$string .= "\n" if (defined(getGlobal("preExec")));
$string .= getGlobal('preExec') . "\n" if (defined(getGlobal("preExec")));
$string .= "\n" if (defined(getGlobal("preExec")));
$string .= "# Paths to things we run.\n";
$string .= "\n";
$string .= "# Path to Canu.\n";
$string .= "bin=\"$FindBin::RealBin\"\n";
$string .= "\n";
$string .= "bin=\"$idir\"\n";
$string .= "pn=" . getGlobal("perl") . "\n";
$string .= "pe=" . "`command -v \$pn`" . "\n";
$string .= "pv=" . "`command \$pn --version | grep version`" . "\n";
$string .= "\n";
$string .= "jn=" . getGlobal("java") . "\n";
$string .= "je=" . "`command -v \$jn`" . "\n";
$string .= "jv=" . "`command \$jn -showversion 2>&1 | head -n 1`" . "\n";
$string .= "\n";
$string .= "cn=" . "$FindBin::RealBin/" . basename($0) . "\n";
$string .= "ce=" . "`command -v \$cn`" . "\n";
$string .= "cv=" . "`command \$cn -version`" . "\n";
$string .= "\n";
$string .= "# Report paths.\n";
$string .= "\n";
$string .= "echo \"\"\n";
$string .= "echo \"Found perl:\"\n";
$string .= "echo \" \" `which perl`\n";
$string .= "echo \" \" `perl --version | grep version`\n";
$string .= "echo \"Found perl (from '\$pn'):\"\n";
$string .= "echo \" \$pe\"\n";
$string .= "echo \" \$pv\"\n";
$string .= "echo \"\"\n";
$string .= "echo \"Found java:\"\n";
$string .= "echo \" \" `which $javaPath`\n";
$string .= "echo \" \" `$javaPath -showversion 2>&1 | head -n 1`\n";
$string .= "echo \"Found java (from '\$jn'):\"\n";
$string .= "echo \" \$je\"\n";
$string .= "echo \" \$jv\"\n";
$string .= "echo \"\"\n";
$string .= "echo \"Found canu:\"\n";
$string .= "echo \" \" $canu\n";
$string .= "echo \" \" `$canu -version`\n";
$string .= "echo \"Found canu (from '\$cn'):\"\n";
$string .= "echo \" \$ce\"\n";
$string .= "echo \" \$cv\"\n";
$string .= "echo \"\"\n";
$string .= "\n";
$string .= "\n";
$string .= "# Environment for any object storage.\n";
$string .= "\n";
$string .= "export CANU_OBJECT_STORE_CLIENT=" . getGlobal("objectStoreClient") . "\n";
Expand All @@ -462,7 +434,6 @@ sub getBinDirectoryShellCode () {
$string .= "export CANU_OBJECT_STORE_NAMESPACE=" . getGlobal("objectStoreNameSpace") . "\n";
$string .= "export CANU_OBJECT_STORE_PROJECT=" . getGlobal("objectStoreProject") . "\n";
$string .= "\n";
$string .= "\n";

return($string);
}
Expand Down Expand Up @@ -686,7 +657,7 @@ sub submitScript ($$) {
print F "rm -f canu.out\n";
print F "ln -s $scriptOut canu.out\n";
print F "\n";
print F "/usr/bin/env perl \\\n";
print F "\$pe \\\n";
print F "\$bin/" . basename($0) . " " . getCommandLineOptions() . " canuIteration=" . getGlobal("canuIteration") . "\n";
close(F);

Expand Down
5 changes: 2 additions & 3 deletions src/pipelines/canu/OverlapMhap.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,6 @@ sub mhapConfigure ($$$) {
my $memorySize = getGlobal("${tag}mhapMemory") * 0.9;
my $blockPerGb = getGlobal("${tag}MhapBlockSize") / (($numHashes < 768) ? 1 : 2);

my $javaPath = getGlobal("java");
my $javaOpt = "-d64" if (defined(getGlobal("javaUse64Bit")) && getGlobal("javaUse64Bit") == 1);
my $javaMemory = int(getGlobal("${tag}mhapMemory") * 1024 * 0.9 + 0.5);

Expand Down Expand Up @@ -344,7 +343,7 @@ sub mhapConfigure ($$$) {
print F "# So mhap writes its output in the correct spot.\n";
print F "cd ./blocks\n";
print F "\n";
print F "$javaPath $javaOpt -XX:ParallelGCThreads=", getGlobal("${tag}mhapThreads"), " -server -Xms", $javaMemory, "m -Xmx", $javaMemory, "m \\\n";
print F "\$je $javaOpt -XX:ParallelGCThreads=", getGlobal("${tag}mhapThreads"), " -server -Xms", $javaMemory, "m -Xmx", $javaMemory, "m \\\n";
print F " -jar $cygA \$bin/../share/java/classes/mhap-" . getGlobal("${tag}MhapVersion") . ".jar $cygB \\\n";
print F " --repeat-weight 0.9 --repeat-idf-scale 10 -k $merSize \\\n";
print F " --supress-noise 2 \\\n" if (defined(getGlobal("${tag}MhapFilterUnique")) && getGlobal("${tag}MhapFilterUnique") == 1);
Expand Down Expand Up @@ -463,7 +462,7 @@ sub mhapConfigure ($$$) {
}

print F " # Start up the producer.\n";
print F " $javaPath $javaOpt -XX:ParallelGCThreads=", getGlobal("${tag}mhapThreads"), " -server -Xms", $javaMemory, "m -Xmx", $javaMemory, "m \\\n";
print F " \$je $javaOpt -XX:ParallelGCThreads=", getGlobal("${tag}mhapThreads"), " -server -Xms", $javaMemory, "m -Xmx", $javaMemory, "m \\\n";
print F " -jar $cygA \$bin/../share/java/classes/mhap-" . getGlobal("${tag}MhapVersion") . ".jar $cygB \\\n";
print F " --repeat-weight 0.9 --repeat-idf-scale 10 -k $merSize \\\n";
print F " --supress-noise 2 \\\n" if (defined(getGlobal("${tag}MhapFilterUnique")) && getGlobal("${tag}MhapFilterUnique") == 1);
Expand Down

0 comments on commit 86ad90d

Please sign in to comment.