1. mar packaging and build changes - Bug 386760 - directory removal w…

…ith software update. r=nthomas, r=khuey

config/createprecomplete.py

@@ -0,0 +1,64 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+# Creates the precomplete file containing the remove, remove-cc, and rmdir
+# application update instructions which is used to remove files and directories
+# that are no longer present in a complete update. The current working directory
+# is used for the location to enumerate and to create the precomplete file.
+
+import sys
+import os
+
+def get_build_entries(root_path):
+    """ Iterates through the root_path, creating a list for each file and
+        directory. Excludes any path starting with extensions or distribution.
+    """
+    rel_file_path_set = set()
+    rel_dir_path_set = set()
+    for root, dirs, files in os.walk(root_path):
+        for file_name in files:
+            parent_dir_rel_path = root[len(root_path)+1:]
+            rel_path_file = os.path.join(parent_dir_rel_path, file_name)
+            rel_path_file = rel_path_file.replace("\\", "/")
+            if not (rel_path_file.startswith("distribution/") or
+                    rel_path_file.startswith("extensions/")):
+                rel_file_path_set.add(rel_path_file)
+
+        for dir_name in dirs:
+            parent_dir_rel_path = root[len(root_path)+1:]
+            rel_path_dir = os.path.join(parent_dir_rel_path, dir_name)
+            rel_path_dir = rel_path_dir.replace("\\", "/")+"/"
+            if not (rel_path_dir.startswith("distribution/") or
+                    rel_path_dir.startswith("extensions/")):
+                rel_dir_path_set.add(rel_path_dir)
+
+    rel_file_path_list = list(rel_file_path_set)
+    rel_file_path_list.sort(reverse=True)
+    rel_dir_path_list = list(rel_dir_path_set)
+    rel_dir_path_list.sort(reverse=True)
+
+    return rel_file_path_list, rel_dir_path_list
+
+def generate_precomplete():
+    """ Creates the precomplete file containing the remove, remove-cc, and rmdir
+        application update instructions. The current working directory is used
+        for the location to enumerate and to create the precomplete file.
+    """
+    root_path = os.getcwd()
+    rel_file_path_list, rel_dir_path_list = get_build_entries(root_path)
+    precomplete_file_path = os.path.join(root_path,"precomplete")
+    # open in binary mode to prevent OS specific line endings.
+    precomplete_file = open(precomplete_file_path, "wb")
+    for rel_file_path in rel_file_path_list:
+        if rel_file_path.endswith("channel-prefs.js"):
+            precomplete_file.writelines("remove-cc \""+rel_file_path+"\"\n")
+        else:
+            precomplete_file.writelines("remove \""+rel_file_path+"\"\n")
+
+    for rel_dir_path in rel_dir_path_list:
+        precomplete_file.writelines("rmdir \""+rel_dir_path+"\"\n")
+
+    precomplete_file.close()
+
+if __name__ == "__main__":
+    generate_precomplete()

toolkit/mozapps/installer/packager.mk

@@ -23,6 +23,7 @@
 #   Benjamin Smedberg <bsmedberg@covad.net>
 #   Arthur Wiebe <artooro@gmail.com>
 #   Mark Mentovai <mark@moxienet.com>
+#   Robert Strong <robert.bugzilla@gmail.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either of the GNU General Public License Version 2 or later (the "GPL"),
@@ -378,6 +379,8 @@ endif
 MAKE_SDK = $(CREATE_FINAL_TAR) - $(MOZ_APP_NAME)-sdk | bzip2 -vf > $(SDK)
 endif
 
+CREATE_PRECOMPLETE = $(PYTHON) $(MOZILLA_DIR)/config/createprecomplete.py
+
 ifdef MOZ_OMNIJAR
 GENERATE_CACHE ?= true
 
@@ -419,10 +422,11 @@ UNPACK_OMNIJAR	= \
   sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \
   mv components.manifest components
 
-MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && $(INNER_MAKE_PACKAGE)
+MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && \
+	              (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE) && (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(UNPACK_OMNIJAR))
 else
-MAKE_PACKAGE	= $(INNER_MAKE_PACKAGE)
+MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE)
 endif
 
@@ -565,6 +569,7 @@ ifdef MOZ_OMNIJAR
 	@(cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR))
 endif
 	@cp -av $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/. $(DEPTH)/installer-stage/core
+	@(cd $(DEPTH)/installer-stage/core && $(CREATE_PRECOMPLETE))
 ifdef MOZ_OPTIONAL_PKG_LIST
 	@$(NSINSTALL) -D $(DEPTH)/installer-stage/optional
 	$(call PACKAGER_COPY, "$(call core_abspath,$(DIST))",\

tools/update-packaging/common.sh

@@ -14,7 +14,7 @@ MBSDIFF=${MBSDIFF:-mbsdiff}
 # Helper routines
 
 notice() {
-  echo $* 1>&2
+  echo "$*" 1>&2
 }
 
 get_file_size() {
@@ -35,13 +35,23 @@ copy_perm() {
 
 make_add_instruction() {
   f="$1"
+
+  # Used to log to the console
+  if [ $2 ]; then
+    forced=" (forced)"
+  else
+    forced=
+  fi
+
   is_extension=$(echo "$f" | grep -c 'extensions/.*/')
   if [ $is_extension = "1" ]; then
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "     add-if: $f$forced"
     echo "add-if \"$testdir\" \"$f\""
   else
+    notice "        add: $f$forced"
     echo "add \"$f\""
   fi
 }
@@ -54,16 +64,21 @@ make_patch_instruction() {
     # Use the subdirectory of the extensions folder as the file to test
     # before performing this add instruction.
     testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    notice "   patch-if: $f"
     echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
   elif [ $is_search_plugin = "1" ]; then
+    notice "   patch-if: $f"
     echo "patch-if \"$f\" \"$f.patch\" \"$f\""
   else
+    notice "      patch: $f"
     echo "patch \"$f.patch\" \"$f\""
   fi
 }
 
 append_remove_instructions() {
   dir="$1"
+  filev1="$2"
+  filev2="$3"
   if [ -f "$dir/removed-files" ]; then
     prefix=
     listfile="$dir/removed-files"
@@ -73,18 +88,43 @@ append_remove_instructions() {
   fi
   if [ -n "$listfile" ]; then
     # Map spaces to pipes so that we correctly handle filenames with spaces.
-    files=($(cat "$listfile" | tr " " "|"))  
+    files=($(cat "$listfile" | tr " " "|"  | sort -r))
     num_files=${#files[*]}
     for ((i=0; $i<$num_files; i=$i+1)); do
-      # Trim whitespace (including trailing carriage returns)
-      f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
-      # Exclude any blank lines or any lines ending with a slash, which indicate
-      # directories.  The updater doesn't know how to remove entire directories.
+      # Map pipes back to whitespace and remove carriage returns
+      f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
+      # Trim whitespace
+      f=$(echo $f)
+      # Exclude blank lines.
       if [ -n "$f" ]; then
-        if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
-          echo "remove \"$prefix$f\""
-        else
-          notice "ignoring remove instruction for directory: $f"
+        # Exclude comments
+        if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
+          # Normalize the path to the root of the Mac OS X bundle if necessary
+          fixedprefix="$prefix"
+          if [ $prefix ]; then
+            if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
+              if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
+                f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
+                fixedprefix=""
+              else
+                f=$(echo $f | sed -e 's:^\.\.\/::')
+                fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
+              fi
+            fi
+          fi
+          if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
+            notice "      rmdir: $fixedprefix$f"
+            echo "rmdir \"$fixedprefix$f\"" >> $filev2
+          elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
+            # Remove the *
+            f=$(echo "$f" | sed -e 's:\*$::')
+            notice "    rmrfdir: $fixedprefix$f"
+            echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
+          else
+            notice "     remove: $fixedprefix$f"
+            echo "remove \"$fixedprefix$f\"" >> $filev1
+            echo "remove \"$fixedprefix$f\"" >> $filev2
+          fi
         fi
       fi
     done
@@ -101,12 +141,30 @@ list_files() {
   find . -type f \
     ! -name "channel-prefs.js" \
     ! -name "update.manifest" \
+    ! -name "updatev2.manifest" \
+    ! -name "temp-dirlist" \
     ! -name "temp-filelist" \
     | sed 's/\.\/\(.*\)/\1/' \
-    | sort > "temp-filelist"
+    | sort -r > "temp-filelist"
   while read file; do
     eval "${1}[$count]=\"$file\""
     (( count++ ))
   done < "temp-filelist"
   rm "temp-filelist"
 }
+
+# List all directories in the current directory, stripping leading "./"
+list_dirs() {
+  count=0
+
+  find . -type d \
+    ! -name "." \
+    ! -name ".." \
+    | sed 's/\.\/\(.*\)/\1/' \
+    | sort -r > "temp-dirlist"
+  while read dir; do
+    eval "${1}[$count]=\"$dir\""
+    (( count++ ))
+  done < "temp-dirlist"
+  rm "temp-dirlist"
+}

tools/update-packaging/make_full_update.sh

@@ -32,39 +32,72 @@ fi
 
 archive="$1"
 targetdir="$2"
+# Prevent the workdir from being inside the targetdir so it isn't included in
+# the update mar.
+if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+  # Remove the /
+  targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
+fi
 workdir="$targetdir.work"
-manifest="$workdir/update.manifest"
-targetfiles="update.manifest"
+updatemanifestv1="$workdir/update.manifest"
+updatemanifestv2="$workdir/updatev2.manifest"
+targetfiles="update.manifest updatev2.manifest"
 
 mkdir -p "$workdir"
 
+# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
+# Info.plist so it launches the right architecture, bug 600098
+
 # Generate a list of all files in the target directory.
 pushd "$targetdir"
 if test $? -ne 0 ; then
   exit 1
 fi
 
-# On Mac, force a top-level file so that OS X reloads the Info.plist
-# and launches the right architecture for the OS version, bug 600098
-if [[ -d Contents ]]; then
-  touch force_plist_reload
+if [ ! -f "precomplete" ]; then
+  notice "precomplete file is missing!"
+  exit 1
 fi
 
 list_files files
 
+# Files that should be added on channel change
+ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
+
 popd
 
-> $manifest
+notice ""
+notice "Adding file add instructions to file 'update.manifest'"
+> $updatemanifestv1
 
 num_files=${#files[*]}
 
 for ((i=0; $i<$num_files; i=$i+1)); do
   f="${files[$i]}"
 
-  notice "processing $f"
+  make_add_instruction "$f" >> $updatemanifestv1
 
-  make_add_instruction "$f" >> $manifest
+  dir=$(dirname "$f")
+  mkdir -p "$workdir/$dir"
+  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  copy_perm "$targetdir/$f" "$workdir/$f"
+
+  targetfiles="$targetfiles \"$f\""
+done
 
+# Add the type of update to the beginning of and cat the contents of the version
+# 1 update manifest to the version 2 update manifest.
+> $updatemanifestv2
+notice ""
+notice "Adding type instruction to file 'updatev2.manifest'"
+notice "       type: complete"
+echo "type \"complete\"" >> $updatemanifestv2
+
+notice ""
+notice "Adding file ADD instructions for channel change to file 'updatev2.manifest'"
+for f in $ccfiles; do
+  notice "     add-cc: $f"
+  echo "add-cc \"$f\"" >> $updatemanifestv2
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
   $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
@@ -73,13 +106,23 @@ for ((i=0; $i<$num_files; i=$i+1)); do
   targetfiles="$targetfiles \"$f\""
 done
 
+notice ""
+notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
+cat $updatemanifestv1 >> $updatemanifestv2
+
 # Append remove instructions for any dead files.
-append_remove_instructions "$targetdir" >> $manifest
+notice ""
+notice "Adding file and directory remove instructions from file 'removed-files'"
+append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
 
-$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
+$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
 
 # cleanup
 rm -fr "$workdir"
+
+notice ""
+notice "Finished"