From f86df118698a535b2ab1baf81ffc7577e53b2d71 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Mon, 2 Sep 2019 14:33:55 +0100 Subject: [PATCH 01/10] Initial version of container code --- Dockerfile | 77 ++++++++++++++++++++++++++++++++++ build/.opt-build-local.sh.swp | Bin 0 -> 12288 bytes build/opt-build-local.sh | 57 +++++++++++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 Dockerfile create mode 100644 build/.opt-build-local.sh.swp create mode 100755 build/opt-build-local.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b3b592e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,77 @@ +FROM quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder + +USER root + +RUN apt-get -yq update +RUN apt-get install -yq --no-install-recommends \ +locales \ +g++ \ +make \ +gcc \ +pkg-config \ +python \ +zlib1g-dev + +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +ENV OPT /opt/wtsi-cgp +ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH +ENV PERL5LIB $OPT/lib/perl5 +ENV LD_LIBRARY_PATH $OPT/lib +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 + +# build tools from other repos +# no need to do other repos it is already installed in base image + +# build the tools in this repo, separate to reduce build time on errors +COPY . . +RUN bash build/opt-build-local.sh $OPT + +FROM ubuntu:16.04 + +LABEL maintainer="cgphelp@sanger.ac.uk" \ + uk.ac.sanger.cgp="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" \ + version="v1.0.0" \ + description="alleleCount docker" + +RUN apt-get -yq update +RUN apt-get install -yq --no-install-recommends \ +apt-transport-https \ +locales \ +curl \ +ca-certificates \ +libperlio-gzip-perl \ +bzip2 \ +psmisc \ +time \ +zlib1g \ +liblzma5 \ +libncurses5 \ +p11-kit \ +unattended-upgrades && \ +unattended-upgrade -d -v && \ +apt-get remove -yq unattended-upgrades && \ +apt-get autoremove -yq + +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +ENV OPT /opt/wtsi-cgp +ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH +ENV PERL5LIB $OPT/lib/perl5 +ENV LD_LIBRARY_PATH $OPT/lib +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 + +RUN mkdir -p $OPT +COPY --from=builder $OPT $OPT + +## USER CONFIGURATION +RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu + +USER ubuntu +WORKDIR /home/ubuntu + +CMD ["/bin/bash"] diff --git a/build/.opt-build-local.sh.swp b/build/.opt-build-local.sh.swp new file mode 100644 index 0000000000000000000000000000000000000000..f15065311fee38bdb02660cc27235d4f3f366f64 GIT binary patch literal 12288 zcmeI2&1)n@7>8>`5!6*t5IiVvW`>QTJvHrqu$wSQoUHC>5{5~l$hxL$x@$5;`pfR> zZ2VZoi|AQc@t}W$iWh|y@8Z#$XJPlCsNltmzv}6k47-}_;%T889;Umx>aBX8s%i+S zLF?u2DxGe+4A+B<$w#9LU+kP?mr#%iw=6cV4!R;WaNXl4S|`*&wHSNX1mqW!Y=?_5 zl%eb-MXdRtP+`E6R6FIJ6DGb0oB8PAVD_ywU=7^7fkF?R(?{9yXWW))RIkm^!w+rV ze0y8d8n6be0c*e-um-FFYrqbnVhM9k6ia#J=z~@z#6aytO0Al z8n6be0c*e-um-FFYrq<~4GoBZu{-Yp_hRt)|NrFg|34pK><@4aTm?UXE8tu34fqOt z2`+fkQ$_ff|F0>6W+;1}>6 zxB@-{%iwA7ICu>Fd>>;!f%icNTthCOgHOO^@B!Ea&x0no8<>1QM&3POFKfUWum-FF zYrq^RpFR88%nRR_d5&s+D&mNoU2=xE-pYv+ zN5xuZ!q;k7l7j**bB&uwWm%HZNb7WgbJI&Rl3J8ENj276=&;Fisf!fHQ>oEiTi@8~ zEp41L!w<}Jcf(}J)R4~8969okwCTbk>5+_CyIl0n( zuI)*Gl+1L@Lp9*3%)*lkjY%bXhfE1MsigUwFTB>^d~x2J#9CciGef>HR3zes{gik< z5}D*>Vb$c2!2cd~pb*u(I;8c=@)i>8FZEw7jovssWV+C}uGW9WH1?hDQxK| z#c6G`9BQj#kNhM`RVcJdV(RoZb0nvcG&S&0W962@UE5_c%q7-qSwg02`xG9_6N^r4}*+FguUGI7OO5&p-0B99x0giGnHyKmMr&_CKpD> zQ_~Bj$R)wat_mb!(N&`vkFJb^Z{UbTPUZ+a?{;43)W#o>TBBYgC&Xm9m<*~u6_ZcH vxVw$j9zHm2ZdQYtr}N~zMYYD; Date: Mon, 2 Sep 2019 16:30:47 +0100 Subject: [PATCH 02/10] Fixed bug in the declaration of the prefix variable --- c/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/Makefile b/c/Makefile index 009e8bb..d204bac 100644 --- a/c/Makefile +++ b/c/Makefile @@ -12,7 +12,7 @@ CFLAGS = -g -Wall HTSLOC?=$(HTSLIB) HTSTMP?=./htslib_tmp -prefix=?/usr/local/ +prefix?=/usr/local/ #Define locations of header files OPTINC?=-I$(HTSLOC)/ From ba5a955f044ee9d3952902683ce710f7c17f0060 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Mon, 2 Sep 2019 16:32:53 +0100 Subject: [PATCH 03/10] removed swap file that shouldn't be in repo --- build/.opt-build-local.sh.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 build/.opt-build-local.sh.swp diff --git a/build/.opt-build-local.sh.swp b/build/.opt-build-local.sh.swp deleted file mode 100644 index f15065311fee38bdb02660cc27235d4f3f366f64..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&1)n@7>8>`5!6*t5IiVvW`>QTJvHrqu$wSQoUHC>5{5~l$hxL$x@$5;`pfR> zZ2VZoi|AQc@t}W$iWh|y@8Z#$XJPlCsNltmzv}6k47-}_;%T889;Umx>aBX8s%i+S zLF?u2DxGe+4A+B<$w#9LU+kP?mr#%iw=6cV4!R;WaNXl4S|`*&wHSNX1mqW!Y=?_5 zl%eb-MXdRtP+`E6R6FIJ6DGb0oB8PAVD_ywU=7^7fkF?R(?{9yXWW))RIkm^!w+rV ze0y8d8n6be0c*e-um-FFYrqbnVhM9k6ia#J=z~@z#6aytO0Al z8n6be0c*e-um-FFYrq<~4GoBZu{-Yp_hRt)|NrFg|34pK><@4aTm?UXE8tu34fqOt z2`+fkQ$_ff|F0>6W+;1}>6 zxB@-{%iwA7ICu>Fd>>;!f%icNTthCOgHOO^@B!Ea&x0no8<>1QM&3POFKfUWum-FF zYrq^RpFR88%nRR_d5&s+D&mNoU2=xE-pYv+ zN5xuZ!q;k7l7j**bB&uwWm%HZNb7WgbJI&Rl3J8ENj276=&;Fisf!fHQ>oEiTi@8~ zEp41L!w<}Jcf(}J)R4~8969okwCTbk>5+_CyIl0n( zuI)*Gl+1L@Lp9*3%)*lkjY%bXhfE1MsigUwFTB>^d~x2J#9CciGef>HR3zes{gik< z5}D*>Vb$c2!2cd~pb*u(I;8c=@)i>8FZEw7jovssWV+C}uGW9WH1?hDQxK| z#c6G`9BQj#kNhM`RVcJdV(RoZb0nvcG&S&0W962@UE5_c%q7-qSwg02`xG9_6N^r4}*+FguUGI7OO5&p-0B99x0giGnHyKmMr&_CKpD> zQ_~Bj$R)wat_mb!(N&`vkFJb^Z{UbTPUZ+a?{;43)W#o>TBBYgC&Xm9m<*~u6_ZcH vxVw$j9zHm2ZdQYtr}N~zMYYD; Date: Mon, 2 Sep 2019 16:33:48 +0100 Subject: [PATCH 04/10] working version of the container (apparently) --- Dockerfile | 5 ++++- build/opt-build-local.sh | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b3b592e..0689d11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,10 @@ make \ gcc \ pkg-config \ python \ -zlib1g-dev +zlib1g-dev \ +libbz2-dev \ +liblzma-dev + RUN locale-gen en_US.UTF-8 RUN update-locale LANG=en_US.UTF-8 diff --git a/build/opt-build-local.sh b/build/opt-build-local.sh index 1546b9b..daec376 100755 --- a/build/opt-build-local.sh +++ b/build/opt-build-local.sh @@ -52,6 +52,10 @@ set -u ### alleleCount if [ ! -e $SETUP_DIR/alleleCount.success ]; then cd $INIT_DIR - ./setup.sh $INST_PATH - touch $SETUP_DIR/alleleCount.success + mkdir -p $INIT_DIR/c/bin + make -C c clean + export prefix=$INST_PATH + make -C c -j$CPU + cp $INIT_DIR/c/bin/alleleCounter $INST_PATH/bin/. + touch $SETUP_DIR/alleleCounter.success fi From 3bc4e9d7dc8b22a079dffddbb52f2644900606a7 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Thu, 5 Sep 2019 13:31:43 +0100 Subject: [PATCH 05/10] Update README.md added the Quay.io badge --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 411950a..6f23fd9 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ The alleleCount package primarily exists to prevent code duplication between some other projects, specifically AscatNGS and Battenberg. +[![Quay Badge][quay-status]][quay-repo] + | Master | Develop | | --------------------------------------------- | ----------------------------------------------- | | [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] | @@ -160,3 +162,8 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008, [ds-cgpwgs-git]: https://github.com/cancerit/dockstore-cgpwgs + + +[quay-status]: https://quay.io/repository/wtsicgp/allelecount/status +[quay-repo]: https://quay.io/repository/wtsicgp/allelecount +[quay-builds]: https://quay.io/repository/wtsicgp/allelecount?tab=builds From 639ba848d60795921d44a5f1cfe7ddb8be202d8a Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Thu, 5 Sep 2019 13:32:14 +0000 Subject: [PATCH 06/10] Added the perl wrapper to the container --- build/opt-build-local.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build/opt-build-local.sh b/build/opt-build-local.sh index daec376..dc677c3 100755 --- a/build/opt-build-local.sh +++ b/build/opt-build-local.sh @@ -49,13 +49,22 @@ export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/ export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'` set -u +CPANM=`which cpanm` ### alleleCount if [ ! -e $SETUP_DIR/alleleCount.success ]; then + #build the C part cd $INIT_DIR mkdir -p $INIT_DIR/c/bin make -C c clean export prefix=$INST_PATH make -C c -j$CPU cp $INIT_DIR/c/bin/alleleCounter $INST_PATH/bin/. + #build the perl part + $CPANM --no-interactive --notest --mirror http://cpan.metacpan.org -l $INST_PATH/ --installdeps $INIT_DIR/perl/. < /dev/null + cd $INIT_DIR/perl + perl Makefile.PL INSTALL_BASE=$INST_PATH + make + make test + make install touch $SETUP_DIR/alleleCounter.success fi From bd371ee09fcea705536021cade9c4af027dbbfb5 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Thu, 5 Sep 2019 13:33:01 +0000 Subject: [PATCH 07/10] added docker image build check --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7c762c3..a2c3b73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,3 +28,6 @@ script: - ./setup.sh $HOME/wtsi-opt - ls -l $HOME/wtsi-opt/bin/alleleCounter - ls -l $HOME/wtsi-opt/bin/alleleCounter.pl + - echo 'Build and check docker image' + - docker build -t allelec . + - docker images | grep -c allelec From 96fbdcffe809e6913e4fb77c168c4fd8497b6405 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Thu, 5 Sep 2019 14:22:44 +0000 Subject: [PATCH 08/10] Updated so that all CI tests get done with container --- .travis.yml | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index a2c3b73..c63bfbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,32 +2,17 @@ notifications: slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR email: false -env: - - CC=gcc +sudo: false -addons: - apt: - packages: - - build-essential - - autoconf - - curl - - zlib1g-dev - - libncurses5-dev - - libpstreams-dev - - libcurl4-openssl-dev +language: ruby -install: true - -language: perl - -perl: - - "5.22" -# - "5.16" +services: + - docker script: - - ./setup.sh $HOME/wtsi-opt - - ls -l $HOME/wtsi-opt/bin/alleleCounter - - ls -l $HOME/wtsi-opt/bin/alleleCounter.pl - echo 'Build and check docker image' - docker build -t allelec . - docker images | grep -c allelec + - echo 'Verify program(s) from this repo' + - docker run -t --rm allelec alleleCounter -v + - docker run -t --rm allelec alleleCounter.pl -v From 9579f6c303523af28880d07709e4d67a292cdcf7 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Sat, 7 Sep 2019 17:16:13 +0000 Subject: [PATCH 09/10] Changed the base image to be directly Ubuntu:16, built all the required dependencies myself --- Dockerfile | 30 ++++++++++---- build/opt-build-local.sh | 8 ++-- build/opt-build.sh | 86 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 11 deletions(-) create mode 100755 build/opt-build.sh diff --git a/Dockerfile b/Dockerfile index 0689d11..68c6031 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,32 @@ -FROM quay.io/wtsicgp/dockstore-cgpmap:3.1.4 as builder +FROM ubuntu:16.04 as builder USER root +# ALL tool versions used by opt-build.sh +ENV VER_HTSLIB="1.7" + RUN apt-get -yq update RUN apt-get install -yq --no-install-recommends \ -locales \ -g++ \ +build-essential \ +apt-transport-https \ +curl \ +ca-certificates \ make \ +bzip2 \ gcc \ -pkg-config \ -python \ +locales \ +curl \ +wget \ +libtasn1-dev \ +libgnutls-dev \ +nettle-dev \ +libgmp-dev \ +libp11-kit-dev \ zlib1g-dev \ libbz2-dev \ -liblzma-dev - +liblzma-dev \ +libcurl4-gnutls-dev \ +libncurses5-dev RUN locale-gen en_US.UTF-8 RUN update-locale LANG=en_US.UTF-8 @@ -26,7 +39,8 @@ ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 # build tools from other repos -# no need to do other repos it is already installed in base image +ADD build/opt-build.sh build/ +RUN bash build/opt-build.sh $OPT # build the tools in this repo, separate to reduce build time on errors COPY . . diff --git a/build/opt-build-local.sh b/build/opt-build-local.sh index dc677c3..3f2e58a 100755 --- a/build/opt-build-local.sh +++ b/build/opt-build-local.sh @@ -47,10 +47,13 @@ export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$// export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'` export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'` export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'` -set -u - CPANM=`which cpanm` +echo "Installing Perl prerequisites ..." +$CPANM --no-interactive --notest --mirror http://cpan.metacpan.org -l $INST_PATH/ --installdeps $INIT_DIR/perl/. < /dev/null + +set -u ### alleleCount +echo "Building alleleCounter ..." if [ ! -e $SETUP_DIR/alleleCount.success ]; then #build the C part cd $INIT_DIR @@ -60,7 +63,6 @@ if [ ! -e $SETUP_DIR/alleleCount.success ]; then make -C c -j$CPU cp $INIT_DIR/c/bin/alleleCounter $INST_PATH/bin/. #build the perl part - $CPANM --no-interactive --notest --mirror http://cpan.metacpan.org -l $INST_PATH/ --installdeps $INIT_DIR/perl/. < /dev/null cd $INIT_DIR/perl perl Makefile.PL INSTALL_BASE=$INST_PATH make diff --git a/build/opt-build.sh b/build/opt-build.sh new file mode 100755 index 0000000..adf235d --- /dev/null +++ b/build/opt-build.sh @@ -0,0 +1,86 @@ +#! /bin/bash + +set -xe + +if [[ -z "${TMPDIR}" ]]; then + TMPDIR=/tmp +fi + +set -u + +if [ "$#" -lt "1" ] ; then + echo "Please provide an installation path such as /opt/ICGC" + exit 1 +fi + +# get path to this script +SCRIPT_PATH=`dirname $0`; +SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)` + +# get the location to install to +INST_PATH=$1 +mkdir -p $1 +INST_PATH=`(cd $1 && pwd)` +echo $INST_PATH + +# get current directory +INIT_DIR=`pwd` + +CPU=`grep -c ^processor /proc/cpuinfo` +if [ $? -eq 0 ]; then + if [ "$CPU" -gt "6" ]; then + CPU=6 + fi +else + CPU=1 +fi +echo "Max compilation CPUs set to $CPU" + +SETUP_DIR=$INIT_DIR/install_tmp +mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end +mkdir -p $INST_PATH/bin +cd $SETUP_DIR + +# make sure tools installed can see the install loc of libraries +set +u +export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'` +export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'` +export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'` +export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'` +set -u + +## grab cpanm +curl -L http://cpanmin.us | perl - App::cpanminus +CPANM=`which cpanm` + +echo "Installing Perl base deps ..." +if [ ! -e $SETUP_DIR/basePerlDeps.success ]; then + perlmods=( "ExtUtils::CBuilder" "Module::Build~0.42" "Const::Fast" "File::Which" "LWP::UserAgent") + for i in "${perlmods[@]}" ; do + $CPANM --no-interactive --notest --mirror http://cpan.metacpan.org -l $INST_PATH $i + done + touch $SETUP_DIR/basePerlDeps.success +fi + +SOURCE_HTSLIB="https://github.com/samtools/htslib/releases/download/${VER_HTSLIB}/htslib-${VER_HTSLIB}.tar.bz2" + +cd $SETUP_DIR + +echo "Downloading htslib ..." +if [ ! -e $SETUP_DIR/htslibGet.success ]; then + cd $SETUP_DIR + wget $SOURCE_HTSLIB + touch $SETUP_DIR/htslibGet.success +fi + +echo "Building htslib ..." +if [ ! -e $SETUP_DIR/htslib.success ]; then + mkdir -p htslib + tar --strip-components 1 -C htslib -jxf htslib-${VER_HTSLIB}.tar.bz2 + cd htslib + ./configure --enable-plugins --enable-libcurl --prefix=$INST_PATH + make -j$CPU + make install + cd $SETUP_DIR + touch $SETUP_DIR/htslib.success +fi From 2bc08873a76d110dc407da00638de8c6a1b01e50 Mon Sep 17 00:00:00 2001 From: Raul Alcantara Date: Mon, 9 Sep 2019 16:51:49 +0100 Subject: [PATCH 10/10] Updated version to 4.1.0 --- CHANGES.md | 4 ++++ perl/lib/Sanger/CGP/AlleleCount.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e35273d..1cb921d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # CHANGES +## v4.1.0 + +* Created Docker file and build scripts to generate a containeraized code + ## v4.0.2 * Added checking of iterator error codes when calling sam_itr_next diff --git a/perl/lib/Sanger/CGP/AlleleCount.pm b/perl/lib/Sanger/CGP/AlleleCount.pm index 32dd689..47593d3 100644 --- a/perl/lib/Sanger/CGP/AlleleCount.pm +++ b/perl/lib/Sanger/CGP/AlleleCount.pm @@ -25,7 +25,7 @@ package Sanger::CGP::AlleleCount; use strict; use base 'Exporter'; -our $VERSION = '4.0.2'; +our $VERSION = '4.1.0'; our @EXPORT = qw($VERSION); 1;