make-dist

#!/bin/sh
### make-dist: create an Emacs distribution tar file from current srcdir

## Copyright (C) 1995, 1997-1998, 2000-2017 Free Software Foundation,
## Inc.

## This file is part of GNU Emacs.

## GNU Emacs is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.

## GNU Emacs is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.

## You should have received a copy of the GNU General Public License
## along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.

### Commentary:

## This basically creates a duplicate directory structure, and then
## hard links into it only those files that should be distributed.
## This means that if you add a file with an odd name, you should make
## sure that this script will include it.

### Code:

progname="$0"

### Exit if a command fails.
#set -e

### Print out each line we read, for debugging's sake.
#set -v

LANGUAGE=C
LC_ALL=C
LC_MESSAGES=
LANG=
export LANGUAGE LC_ALL LC_MESSAGES LANG

## Remove unnecessary restrictions on file access.
umask 022

update=yes
check=yes
clean_up=no
make_tar=no
default_gzip=gzip
newer=""
with_tests=no
changelog=yes
verbose=no

while [ $# -gt 0 ]; do
  case "$1" in
    ## This option tells make-dist to delete the staging directory
    ## when done.  It is useless to use this unless you make a tar file.
    "--clean-up" )
      clean_up=yes
    ;;
    ## This option tells make-dist to make a tar file.
    "--tar" )
      make_tar=yes
    ;;
    ## This option tells make-dist not to recompile or do analogous things.
    "--no-update" )
      update=no
    ;;
    ## This option says don't check for bad file names, etc.
    "--no-check" )
      check=no
    ;;
    "--no-changelog" )
      changelog=no
    ;;
    ## This option tells make-dist to make the distribution normally, then
    ## remove all files older than the given timestamp file.  This is useful
    ## for creating incremental or patch distributions.
    "--newer")
      newer="$2"
      new_extension=".new"
      shift
    ;;
    ## This option tells make-dist to use 'bzip2' instead of gzip.
    "--bzip2")
      default_gzip="bzip2"
    ;;
    ## Same with xz.
    "--xz")
      default_gzip="xz"
    ;;
    "--no-compress")
      default_gzip="cat"
    ;;

    "--snapshot")
      clean_up=yes
      make_tar=yes
      update=no
     ;;

    ## Include the test/ directory.
    ## This option is mainly for the hydra build server.
    "--tests")
      with_tests=yes
     ;;

    "--verbose")
      verbose=yes
     ;;

    "--help")
      printf '%s\n' "Usage: ${progname} [options]"
      echo ""
      echo "  --bzip2	use bzip2 instead of gzip"
      echo "  --clean-up	delete staging directories when done"
      echo "  --xz		use xz instead of gzip"
      echo "  --no-compress	don't compress"
      echo "  --newer=TIME	don't include files older than TIME"
      echo "  --no-check	don't check for bad file names etc."
      echo "  --no-update	don't recompile or do analogous things"
      echo "  --no-changelog	don't generate the top-level ChangeLog"
      echo "  --snapshot	same as --clean-up --no-update --tar --no-check"
      echo "  --tar		make a tar file"
      echo "  --tests	include the test/ directory"
      echo "  --verbose		noisier output"
      echo ""
      exit 0
    ;;

    * )
      printf '%s\n' "${progname}: Unrecognized argument: $1" >&2
      exit 1
    ;;
  esac
  shift
done

### Make sure we're running in the right place.
if [ ! -d src -o ! -f src/lisp.h -o ! -d lisp -o ! -f lisp/subr.el ]; then
  printf '%s\n' "${progname}: Can't find 'src/lisp.h' and 'lisp/subr.el'." >&2
  printf '%s\n' "${progname} must be run in the top directory of the Emacs" >&2
  printf '%s\n' "distribution tree.  cd to that directory and try again." >&2
  exit 1
fi

### Find where to run Emacs.
### (Accept only absolute file names.)
if [ $update = yes ];
then
  if [ -f src/emacs ];
  then
    EMACS=`pwd`/src/emacs
  else
    case $EMACS in
      /*) ;;
      *)
	if [ ! -f "$EMACS" ]; then
	  printf '%s\n' "$0: You must set the EMACS environment variable " \
	       "to an absolute file name." 2>&1
	  exit 1
	fi;;
    esac
  fi
fi

### Find out which version of Emacs this is.
version=`
  sed -n 's/^AC_INIT(GNU Emacs,[	 ]*\([^	 ,)]*\).*/\1/p' <configure.ac
` || version=
if [ ! "${version}" ]; then
  printf '%s\n' \
    "${progname}: can't find current Emacs version in './src/emacs.c'" >&2
  exit 1
fi

echo Version number is $version

if [ $update = yes ]; then
  if ! grep -q "tree holds version  *${version}" README; then
    echo "WARNING: README has the wrong version number"
    echo "Consider running M-x set-version from admin/admin.el"
    sleep 5
  fi
fi

### Make sure we don't already have a directory  emacs-${version}.

emacsname="emacs-${version}${new_extension}"

if [ -d ${emacsname} ]
then
  echo Directory "${emacsname}" already exists >&2
  exit 1
fi

### Make sure the subdirectory is available.
tempparent="make-dist.tmp.$$"
if [ -d ${tempparent} ]; then
  printf '%s\n' "${progname}: staging directory '${tempparent}' already exists.
Perhaps a previous invocation of '${progname}' failed to clean up after
itself.  Check that directories whose names are of the form
'make-dist.tmp.NNNNN' don't contain any important information, remove
them, and try again." >&2
  exit 1
fi

if [ $check = yes ]; then

  echo "Sanity checking (use --no-check to disable this)..."

  error=no

  ls -1 lisp/[a-zA-Z]*.el lisp/[a-z]*/[a-zA-Z0-9]*.el \
	lisp/[a-z]*/[a-z]*/[a-zA-Z0-9]*.el \
	lisp/[a-z]*/[a-z]*/[a-z]*/[a-zA-Z0-9]*.el > /tmp/el

  ls -1 lisp/[a-zA-Z]*.elc lisp/[a-z]*/[a-zA-Z0-9]*.elc \
	lisp/[a-z]*/[a-z]*/[a-zA-Z0-9]*.elc \
	lisp/[a-z]*/[a-z]*/[a-z]*/[a-zA-Z0-9]*.elc > /tmp/elc

  ## Check for .elc files with no corresponding .el file.
  sed 's/\.el$/.elc/' /tmp/el > /tmp/elelc

  bogosities=`comm -13 /tmp/elelc /tmp/elc`
  if [ x"${bogosities}" != x"" ]; then
    error=yes
    echo "The following .elc files have no corresponding .el files:"
    echo "${bogosities}"
  fi

  ### Check for .el files with no corresponding .elc file.
  sed 's/\.elc$/.el/' /tmp/elc > /tmp/elcel
  losers=`comm -23 /tmp/el /tmp/elcel`

  bogosities=
  while read elc; do
      el=`echo $elc | sed 's/c$//'`
      [ -e $el ] || continue
      [ $elc -nt $el ] || bogosities="$bogosities $elc"
  done < /tmp/elc

  if [ x"${bogosities}" != x"" ]; then
      error=yes
      echo "The following .elc files are older than their .el files:"
      echo "${bogosities}"
  fi

  rm -f /tmp/el /tmp/elc /tmp/elcel /tmp/elelc

  bogosities=
  for file in $losers; do
    grep -q "no-byte-compile: t" $file && continue
    case $file in
      site-init.el | site-load.el | site-start.el | default.el) continue ;;
    esac

    bogosities="$file $bogosities"

  done
  if [ x"${bogosities}" != x"" ]; then
    error=yes
    echo "The following .el files have no corresponding .elc files:"
    echo "${bogosities}"
  fi


  ## This is only a crude check, eg it does not handle .info
  ## files with multiple .texi source files.
  find doc -name '*.texi' > /tmp/el

  bogosities=
  while read texi; do
      info=`sed -n 's/^@setfilename //p' $texi | sed 's|.*info/||'`
      [ x"${info}" != x"" ] || continue
      info=info/$info
      [ -e $info ] || continue
      [ $info -nt $texi ] || bogosities="$bogosities $info"
  done < /tmp/el

  rm -f /tmp/el

  if [ x"${bogosities}" != x"" ]; then
    error=yes
    echo "The following .info files are older than their .texi files:"
    echo "${bogosities}"
  fi

  ## This exits with non-zero status if any .info files need
  ## rebuilding.
  if [ -e Makefile ]; then
      echo "Checking to see if info files are up-to-date..."
      make --question info || error=yes
  fi

  if [ $error = yes ]; then
      echo "Failed checks" >&2
      exit 1
  fi

fi

if [ $update = yes ]; then

  ## Make sure configure is newer than configure.ac, etc.
  ## It is better to let autoreconf do what is needed than
  ## for us to try and duplicate all its checks.
  echo "Running autoreconf"
  autoreconf -i -I m4 || { x=$?; echo Autoreconf FAILED! >&2; exit $x; }

  ## Make sure src/stamp-h.in is newer than configure.ac.
  rm -f src/stamp-h.in
  echo timestamp > src/stamp-h.in

  echo "Updating Info files"
  make info

  echo "Updating finder, custom and autoload data"
  (cd lisp && make updates EMACS="$EMACS")

  echo "Updating leim-list.el"
  (cd leim && make leim-list.el EMACS="$EMACS")

  echo "Recompiling Lisp files"
  $EMACS -batch -f batch-byte-recompile-directory lisp
fi                              # $update = yes

echo "Creating staging directory: '${tempparent}'"

mkdir ${tempparent}
tempdir="${tempparent}/${emacsname}"

### This trap ensures that the staging directory will be cleaned up even
### when the script is interrupted in mid-career.
if [ "${clean_up}" = yes ]; then
  trap "echo 'Cleaning up the staging directory'; rm -rf ${tempparent}" EXIT
fi

echo "Creating top directory: '${tempdir}'"
mkdir ${tempdir}

if [ "$changelog" = yes ]; then
  if test -e .git; then
    echo "Making top-level ChangeLog"
    make ChangeLog CHANGELOG=${tempdir}/ChangeLog || \
      { x=$?; echo "make ChangeLog FAILED (try --no-changelog?)" >&2; exit $x; }
  else
    echo "No repository, so omitting top-level ChangeLog"
  fi
fi

### We copy in the top-level files before creating the subdirectories in
### hopes that this will make the top-level files appear first in the
### tar file; this means that people can start reading the INSTALL and
### README while the rest of the tar file is still unpacking.  Whoopee.
echo "Making links to top-level files"
ln INSTALL README BUGS ${tempdir}
ln ChangeLog.*[0-9] Makefile.in autogen.sh configure configure.ac ${tempdir}
ln config.bat make-dist .dir-locals.el ${tempdir}
ln aclocal.m4 CONTRIBUTE ${tempdir}

echo "Creating subdirectories"
for subdir in site-lisp \
	      leim leim/CXTERM-DIC leim/MISC-DIC leim/SKK-DIC \
	      build-aux build-aux/snippet \
	      src src/bitmaps lib lib-src oldXMenu lwlib \
	      nt nt/inc nt/inc/sys nt/inc/arpa nt/inc/netinet nt/icons \
	      `find etc lisp admin test -type d` \
	      doc doc/emacs doc/misc doc/man doc/lispref doc/lispintro \
	      info m4 modules modules/mod-test msdos \
	      nextstep nextstep/templates \
	      nextstep/Cocoa nextstep/Cocoa/Emacs.base \
	      nextstep/Cocoa/Emacs.base/Contents \
	      nextstep/Cocoa/Emacs.base/Contents/Resources \
	      nextstep/GNUstep \
	      nextstep/GNUstep/Emacs.base \
	      nextstep/GNUstep/Emacs.base/Resources
do

  if [ "$with_tests" != "yes" ]; then
    case $subdir in
        test*|*/mod-test*) continue ;;
    esac
  fi

  ## site-lisp for in-place installs (?).
  [ "$subdir" = "site-lisp" ] || [ -d "$subdir" ] || \
      echo "WARNING: $subdir not found, making anyway"
  [ "$verbose" = "yes" ] && echo "  ${tempdir}/${subdir}"
  mkdir ${tempdir}/${subdir}
done

echo "Making links to 'lisp' and its subdirectories"
files=`find lisp \( -name '*.el' -o -name '*.elc' -o -name 'ChangeLog*' \
 -o -name 'README' \)`

### Don't distribute site-init.el, site-load.el, or default.el.
for file in lisp/Makefile.in $files; do
  case $file in
    */site-init*|*/site-load*|*/default*) continue ;;
  esac
  ln $file $tempdir/$file
done

echo "Making links to 'leim' and its subdirectories"
(cd leim
 ln ChangeLog.*[0-9] README ../${tempdir}/leim
 ln CXTERM-DIC/README CXTERM-DIC/*.tit ../${tempdir}/leim/CXTERM-DIC
 ln SKK-DIC/README SKK-DIC/SKK-JISYO.L ../${tempdir}/leim/SKK-DIC
 ln MISC-DIC/README MISC-DIC/*.* ../${tempdir}/leim/MISC-DIC
 ln Makefile.in ../${tempdir}/leim/Makefile.in
 ln leim-ext.el ../${tempdir}/leim/leim-ext.el)

## FIXME Can we not just use the "find -type f" method for this one?
echo "Making links to 'build-aux'"
(cd build-aux
 ln compile config.guess config.sub depcomp msys-to-w32 ../${tempdir}/build-aux
 ln gitlog-to-changelog gitlog-to-emacslog ../${tempdir}/build-aux
 ln install-sh missing move-if-change ../${tempdir}/build-aux
 ln update-copyright update-subdirs ../${tempdir}/build-aux
 ln dir_top make-info-dir ar-lib ../${tempdir}/build-aux)

echo "Making links to 'build-aux/snippet'"
(cd build-aux/snippet
 ln *.h ../../${tempdir}/build-aux/snippet)

echo "Making links to 'src'"
### Don't distribute the configured versions of
### config.in, paths.in, buildobj.h, or Makefile.in.
(cd src
 echo "  (It is ok if ln fails in some cases.)"
 ln [a-zA-Z]*.[chm] ../${tempdir}/src
 ln [a-zA-Z]*.in ../${tempdir}/src
 ln deps.mk ../${tempdir}/src
 ln README ChangeLog.*[0-9] ../${tempdir}/src
 ln .gdbinit .dbxinit ../${tempdir}/src
 cd ../${tempdir}/src
 rm -f globals.h config.h epaths.h Makefile buildobj.h)

echo "Making links to 'src/bitmaps'"
(cd src/bitmaps
 ln README *.xbm ../../${tempdir}/src/bitmaps)

echo "Making links to 'lib'"
(snippet_h=`(cd build-aux/snippet && ls *.h)`
 cd lib
 ln [a-zA-Z]*.[ch] ../${tempdir}/lib
 ln gnulib.mk Makefile.am Makefile.in ../${tempdir}/lib
 cd ../${tempdir}/lib
 script='/[*]/d; s/\.in\.h$/.h/'
 rm -f `(echo "$snippet_h"; ls *.in.h) | sed "$script"`)

echo "Making links to 'lib-src'"
(cd lib-src
 ln [a-zA-Z]*.[ch] ../${tempdir}/lib-src
 ln ChangeLog.*[0-9] Makefile.in README ../${tempdir}/lib-src
 ln rcs2log ../${tempdir}/lib-src
 ln update-game-score.exe.manifest ../${tempdir}/lib-src)

echo "Making links to 'm4'"
(cd m4
 ln *.m4 ../${tempdir}/m4)

echo "Making links to 'modules'"
(cd modules
 ln *.py ../${tempdir}/modules
 if [ "$with_tests" = "yes" ]; then
   for f in `find mod-test -type f`; do
       case $f in
           *.log|*.o|*.so) continue ;;
       esac
       ln $f ../$tempdir/modules/$f
   done
 fi
)

echo "Making links to 'nt'"
(cd nt
 ln emacs-x86.manifest emacs-x64.manifest ../${tempdir}/nt
 ln [a-z]*.bat [a-z]*.[ch] ../${tempdir}/nt
 ln *.in gnulib.mk gnulib-modules-to-delete.cfg ../${tempdir}/nt
 ln mingw-cfg.site epaths.nt INSTALL.W64 ../${tempdir}/nt
 ln ChangeLog.*[0-9] INSTALL README README.W32 ../${tempdir}/nt)

echo "Making links to 'nt/inc' and its subdirectories"
for f in `find nt/inc -type f -name '[a-z]*.h'`; do
  ln $f $tempdir/$f
done

echo "Making links to 'nt/icons'"
(cd nt/icons
 ln README [a-z]*.ico ../../${tempdir}/nt/icons
 ln [a-z]*.cur ../../${tempdir}/nt/icons)

echo "Making links to 'msdos'"
(cd msdos
 ln ChangeLog.*[0-9] INSTALL README emacs.ico emacs.pif ../${tempdir}/msdos
 ln depfiles.bat inttypes.h ../${tempdir}/msdos
 ln mainmake.v2 sed*.inp ../${tempdir}/msdos)

echo "Making links to 'nextstep'"
(cd nextstep
 ln ChangeLog.*[0-9] README INSTALL Makefile.in ../${tempdir}/nextstep)

echo "Making links to 'nextstep/templates'"
(cd nextstep/templates
 ln Emacs.desktop.in Info-gnustep.plist.in Info.plist.in InfoPlist.strings.in ../../${tempdir}/nextstep/templates)

echo "Making links to 'nextstep/Cocoa/Emacs.base/Contents'"
(cd nextstep/Cocoa/Emacs.base/Contents
 ln PkgInfo ../../../../${tempdir}/nextstep/Cocoa/Emacs.base/Contents)

echo "Making links to 'nextstep/Cocoa/Emacs.base/Contents/Resources'"
(cd nextstep/Cocoa/Emacs.base/Contents/Resources
 ln Credits.html *.icns ../../../../../${tempdir}/nextstep/Cocoa/Emacs.base/Contents/Resources)

echo "Making links to 'nextstep/GNUstep/Emacs.base/Resources'"
(cd nextstep/GNUstep/Emacs.base/Resources
 ln README emacs.tiff ../../../../${tempdir}/nextstep/GNUstep/Emacs.base/Resources )

echo "Making links to 'oldXMenu'"
(cd oldXMenu
 ln *.[ch] *.in *.mk ../${tempdir}/oldXMenu
 ln README ChangeLog.*[0-9] ../${tempdir}/oldXMenu)

echo "Making links to 'lwlib'"
(cd lwlib
 ln *.[ch] *.in *.mk ../${tempdir}/lwlib
 ln README ChangeLog.*[0-9] ../${tempdir}/lwlib)

## It is important to distribute admin/ because it contains sources
## for generated lisp/international/uni-*.el files.
echo "Making links to 'admin' and its subdirectories"
for f in `find admin -type f`; do
  case $f in
    */Makefile) [ -f $f.in ] && continue ;;
  esac
  ln $f $tempdir/$f
done

if [ "$with_tests" = "yes" ]; then
  echo "Making links to 'test' and its subdirectories"
  for f in `find test -type f`; do
    case $f in
      test/automated/*.log) continue ;;
      test/automated/flymake/warnpred/a.out) continue ;;
      test/automated/Makefile) continue ;;
    esac
    ln $f $tempdir/$f
  done
fi

echo "Making links to 'etc' and its subdirectories"
for f in `find etc -type f`; do
  case $f in
    etc/DOC*|etc/*.pyc) continue ;;
    ## Arguably we should not exclude *.ps.
    etc/refcards/*.aux|etc/refcards/*.dvi|etc/refcards/*.log|etc/refcards/*.ps)
      continue ;;
  esac
  ln $f $tempdir/$f
done

echo "Making links to 'info'"
ln `find info -type f -print` ${tempdir}/info

echo "Making links to 'doc/emacs'"
(cd doc/emacs
 ln *.texi *.in ChangeLog.*[0-9] ../../${tempdir}/doc/emacs)

echo "Making links to 'doc/misc'"
(cd doc/misc
 ln *.texi *.tex *.in gnus-news.el ChangeLog.*[0-9] \
    ../../${tempdir}/doc/misc)

echo "Making links to 'doc/lispref'"
(cd doc/lispref
 ln *.texi *.in README ChangeLog.*[0-9] ../../${tempdir}/doc/lispref
 ln spellfile ../../${tempdir}/doc/lispref
 ln two-volume.make two-volume-cross-refs.txt ../../${tempdir}/doc/lispref)

echo "Making links to 'doc/lispintro'"
(cd doc/lispintro
 ln *.texi *.in *.eps *.pdf ../../${tempdir}/doc/lispintro
 ln README ChangeLog.*[0-9] ../../${tempdir}/doc/lispintro
 cd ../../${tempdir}/doc/lispintro)

echo "Making links to 'doc/man'"
(cd doc/man
 ln *.*[0-9] *.in ../../${tempdir}/doc/man
 cd ../../${tempdir}/doc/man
 rm -f emacs.1)

### It would be nice if they could all be symlinks to top-level copy, but
### you're not supposed to have any symlinks in distribution tar files.
echo "Making sure copying notices are all copies of 'COPYING'"
for subdir in . etc leim lib lib-src lisp lwlib msdos nt src; do
  rm -f ${tempdir}/${subdir}/COPYING
  cp COPYING ${tempdir}/${subdir}
done

if [ "${newer}" ]; then
  printf '%s\n' "Removing files older than $newer"
  ## We remove .elc files unconditionally, on the theory that anyone picking
  ## up an incremental distribution already has a running Emacs to byte-compile
  ## them with.
  find ${tempparent} \( -name '*.elc' -o ! -newer ${newer} \) -exec rm -f {} \;
fi

## Don't distribute backups, autosaves, etc.
echo "Removing unwanted files"
find ${tempparent} \( -name '*~' -o -name '#*#' -o -name '.*ignore' -o -name '=*' -o -name 'TAGS' \) -exec rm -f {} \;

if [ "${make_tar}" = yes ]; then
  echo "Looking for $default_gzip"
  found=0
  temppath=`printf '%s\n' "$PATH" |
    sed -e 's/^:/.:/' -e 's/::/:.:/g' -e 's/:$/:./' -e 's/:/ /g'
  `
  for dir in ${temppath}; do
    [ -x ${dir}/$default_gzip ] || continue
    found=1; break
  done
  if [ "$found" = "0" ]; then
    echo "WARNING: '$default_gzip' not found, will not compress" >&2
    default_gzip=cat
  fi
  case "${default_gzip}" in
    bzip2) gzip_extension=.bz2 ;;
    xz)  gzip_extension=.xz ;;
    gzip)  gzip_extension=.gz ; default_gzip="gzip --best";;
       *)  gzip_extension= ;;
  esac
  echo "Creating tar file"
  taropt=
  [ "$verbose" = "yes" ] && taropt=v

  (cd ${tempparent} ; tar c${taropt}f - ${emacsname} ) \
    | ${default_gzip} \
    > ${emacsname}.tar${gzip_extension}
fi

if [ "${clean_up}" != yes ]; then
  (cd ${tempparent}; mv ${emacsname} ..)
  rm -rf ${tempparent}
fi

### make-dist ends here