[6.0] New tool to update the lists of files and folders to be deleted on core update in script.php

build/deleted_file_check.php

@@ -1,24 +1,30 @@
 <?php
 
 /**
- * This file is used to build the list of deleted files between two reference points.
+ * This file is used to build the lists of deleted files, deleted folders and
+ * renamed files between two Joomla versions.
  *
  * This script requires one parameter:
  *
- * --from - The git commit reference to use as the starting point for the comparison.
+ * --from - Full package zip file or folder with unpacked full package of the
+ *          starting point for the comparison, i.e. the older version.
  *
  * This script has one additional optional parameter:
  *
- * --to - The git commit reference to use as the ending point for the comparison.
+ * --to - Full package zip file or folder with unpacked full package of the
+ *        ending point for the comparison, i.e. the newer version.
  *
- * The reference parameters may be any valid identifier (i.e. a branch, tag, or commit SHA)
+ * If the "to" parameter is not given, the full package zip from a previous
+ * run of the build script is used, if present.
  *
  * @package    Joomla.Build
  *
  * @copyright  (C) 2017 Open Source Matters, Inc. <https://www.joomla.org>
  * @license    GNU General Public License version 2 or later; see LICENSE.txt
  */
 
+use Joomla\CMS\Version;
+
 /*
  * Constants
  */
@@ -28,8 +34,15 @@ function usage($command)
 {
     echo PHP_EOL;
     echo 'Usage: php ' . $command . ' [options]' . PHP_EOL;
-    echo PHP_TAB . '--from <ref>:' . PHP_TAB . 'Starting commit reference (branch/tag)' . PHP_EOL;
-    echo PHP_TAB . '--to <ref>:' . PHP_TAB . 'Ending commit reference (branch/tag) [optional]' . PHP_EOL;
+    echo PHP_TAB . '--from <path>:' . PHP_TAB . 'Path to starting version' . PHP_EOL;
+    echo PHP_TAB . '--to <path>:' . PHP_TAB . 'Path to ending version [optional]' . PHP_EOL;
+    echo PHP_EOL;
+    echo '<path> can be either of the following:' . PHP_EOL;
+    echo PHP_TAB . '- Path to a full package Zip file.' . PHP_EOL;
+    echo PHP_TAB . '- Path to a directory where a full package Zip file has been extracted to.' . PHP_EOL;
+    echo PHP_EOL;
+    echo 'If the "to" parameter is not specified, file "build/tmp/packages/*Full_Package.zip"' . PHP_EOL;
+    echo 'is used if it exists from a previous run of the build script.' . PHP_EOL;
     echo PHP_EOL;
 }
 
@@ -39,108 +52,183 @@ function usage($command)
 
 $options = getopt('', ['from:', 'to::']);
 
-// We need the from reference, otherwise we're doomed to fail
+// We need the "from" parameter, otherwise we're doomed to fail
 if (empty($options['from'])) {
     echo PHP_EOL;
-    echo 'Missing starting directory' . PHP_EOL;
+    echo 'Missing "from" parameter' . PHP_EOL;
 
     usage($argv[0]);
 
     exit(1);
 }
 
-// Missing the to reference?  No problem, grab the current HEAD
+// If the "to" parameter is not specified, use the default
 if (empty($options['to'])) {
+    // Import the version class to get the version information
+    \define('JPATH_PLATFORM', 1);
+    require_once \dirname(__DIR__) . '/libraries/src/Version.php';
+
+    $fullVersion      = (new Version())->getShortVersion();
+    $packageStability = str_replace(' ', '_', Version::DEV_STATUS);
+    $packageFile      = __DIR__ . '/tmp/packages/Joomla_' . $fullVersion . '-' . $packageStability . '-Full_Package.zip';
+
+    if (is_file($packageFile)) {
+        $options['to'] = $packageFile;
+    } else {
+        echo PHP_EOL;
+        echo 'Missing "to" parameter and no zip file "' . $packageFile . '" found.' . PHP_EOL;
+
+        usage($argv[0]);
+
+        exit(1);
+    }
+}
+
+// Check from and to if folder or zip file
+if (!is_dir($options['from']) && !(is_file($options['from']) && substr(strtolower($options['from']), -4) === '.zip')) {
     echo PHP_EOL;
-    echo 'Missing ending directory' . PHP_EOL;
+    echo 'The "from" parameter is neither a directory nor a zip file' . PHP_EOL;
 
-    usage($argv[0]);
+    exit(1);
+}
+
+if (!is_dir($options['to']) && !(is_file($options['to']) && substr(strtolower($options['to']), -4) === '.zip')) {
+    echo PHP_EOL;
+    echo 'The "to" parameter is neither a directory nor a zip file' . PHP_EOL;
 
     exit(1);
 }
 
-// Directories to skip for the check (needs to include anything from previous versions which we want to keep)
-$previousReleaseExclude = [
-    $options['from'] . '/images/sampledata',
-    $options['from'] . '/installation',
-    $options['from'] . '/media/plg_captcha_recaptcha',
-    $options['from'] . '/media/plg_captcha_recaptcha_invisible',
-    $options['from'] . '/media/plg_behaviour_compat',
-    $options['from'] . '/plugins/behaviour/compat',
+// Directories to skip for the check
+$excludedFolders = [
+    'images/sampledata',
+    'installation',
+    'media/plg_captcha_recaptcha',
+    'media/plg_captcha_recaptcha_invisible',
+    'media/plg_behaviour_compat',
+    'plugins/behaviour/compat',
 ];
 
 /**
- * @param   SplFileInfo                      $file      The file being checked
- * @param   mixed                            $key       ?
- * @param   RecursiveCallbackFilterIterator  $iterator  The iterator being processed
+ * @param   string  $folderPath      Path to the folder with the extracted full package
+ * @param   array   $excludeFolders  Excluded folders
  *
- * @return bool True if you need to recurse or if the item is acceptable
+ * @return  stdClass  An object with arrays "files" and "folders"
  */
-$previousReleaseFilter = function ($file, $key, $iterator) use ($previousReleaseExclude) {
-    if ($iterator->hasChildren() && !\in_array($file->getPathname(), $previousReleaseExclude)) {
-        return true;
+function readFolder($folderPath, $excludeFolders): stdClass
+{
+    $return = new stdClass();
+
+    $return->files   = [];
+    $return->folders = [];
+
+    $skipFolders = [];
+
+    foreach ($excludeFolders as $excludeFolder) {
+        $skipFolders[] = $folderPath . '/' . $excludeFolder;
     }
 
-    return $file->isFile();
-};
+    /**
+     * @param   SplFileInfo                      $file      The file being checked
+     * @param   mixed                            $key       ?
+     * @param   RecursiveCallbackFilterIterator  $iterator  The iterator being processed
+     *
+     * @return  bool  True if you need to recurse or if the item is acceptable
+     */
+    $releaseFilter = function ($file, $key, $iterator) use ($skipFolders) {
+        if ($iterator->hasChildren() && !\in_array($file->getPathname(), $skipFolders)) {
+            return true;
+        }
 
-// Directories to skip for the check
-$newReleaseExclude = [
-    $options['to'] . '/installation',
-];
+        return $file->isFile();
+    };
+
+    $releaseDirIterator = new RecursiveDirectoryIterator($folderPath, RecursiveDirectoryIterator::SKIP_DOTS);
+    $releaseIterator    = new RecursiveIteratorIterator(
+        new RecursiveCallbackFilterIterator($releaseDirIterator, $releaseFilter),
+        RecursiveIteratorIterator::SELF_FIRST
+    );
+
+    foreach ($releaseIterator as $info) {
+        if ($info->isDir()) {
+            $return->folders[] = "'" . str_replace($folderPath, '', $info->getPathname()) . "',";
+            continue;
+        }
+
+        $return->files[] = "'" . str_replace($folderPath, '', $info->getPathname()) . "',";
+    }
+
+    return $return;
+}
 
 /**
- * @param   SplFileInfo                      $file      The file being checked
- * @param   mixed                            $key       ?
- * @param   RecursiveCallbackFilterIterator  $iterator  The iterator being processed
+ * @param   string  $filePath        Path to the full package zip file
+ * @param   array   $excludeFolders  Excluded folders
  *
- * @return bool True if you need to recurse or if the item is acceptable
+ * @return  stdClass  An object with arrays "files" and "folders"
  */
-$newReleaseFilter = function ($file, $key, $iterator) use ($newReleaseExclude) {
-    if ($iterator->hasChildren() && !\in_array($file->getPathname(), $newReleaseExclude)) {
-        return true;
-    }
+function readZipFile($filePath, $excludeFolders): stdClass
+{
+    $return = new stdClass();
 
-    return $file->isFile();
-};
+    $return->files   = [];
+    $return->folders = [];
 
-$previousReleaseDirIterator = new RecursiveDirectoryIterator($options['from'], RecursiveDirectoryIterator::SKIP_DOTS);
-$previousReleaseIterator    = new RecursiveIteratorIterator(
-    new RecursiveCallbackFilterIterator($previousReleaseDirIterator, $previousReleaseFilter),
-    RecursiveIteratorIterator::SELF_FIRST
-);
-$previousReleaseFiles   = [];
-$previousReleaseFolders = [];
+    $zipArchive = new ZipArchive();
 
-foreach ($previousReleaseIterator as $info) {
-    if ($info->isDir()) {
-        $previousReleaseFolders[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
-        continue;
+    if ($zipArchive->open($filePath) !== true) {
+        echo PHP_EOL;
+        echo 'Could not open zip archive "' . $filePath . '".' . PHP_EOL;
+
+        exit(1);
     }
 
-    $previousReleaseFiles[] = "'" . str_replace($options['from'], '', $info->getPathname()) . "',";
-}
+    $excludeRegexp = '/^(';
 
-$newReleaseDirIterator = new RecursiveDirectoryIterator($options['to'], RecursiveDirectoryIterator::SKIP_DOTS);
-$newReleaseIterator    = new RecursiveIteratorIterator(
-    new RecursiveCallbackFilterIterator($newReleaseDirIterator, $newReleaseFilter),
-    RecursiveIteratorIterator::SELF_FIRST
-);
-$newReleaseFiles   = [];
-$newReleaseFolders = [];
+    foreach ($excludeFolders as $excludeFolder) {
+        $excludeRegexp .= preg_quote($excludeFolder, '/') . '|';
+    }
 
-foreach ($newReleaseIterator as $info) {
-    if ($info->isDir()) {
-        $newReleaseFolders[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
-        continue;
+    $excludeRegexp = rtrim($excludeRegexp, '|') . ')\/.*/';
+
+    for ($i = 0; $i < $zipArchive->numFiles; $i++) {
+        $stat = $zipArchive->statIndex($i);
+
+        $name = $stat['name'];
+
+        if (preg_match($excludeRegexp, $name) === 1) {
+            continue;
+        }
+
+        if (substr($name, -1) === '/') {
+            $return->folders[] = "'/" . rtrim($name, '/') . "',";
+        } else {
+            $return->files[] = "'/" . $name . "',";
+        }
     }
 
-    $newReleaseFiles[] = "'" . str_replace($options['to'], '', $info->getPathname()) . "',";
+    $zipArchive->close();
+
+    return $return;
+}
+
+// Read files and folders lists from folders or zip files
+if (is_dir($options['from'])) {
+    $previousReleaseFilesFolders = readFolder($options['from'], $excludedFolders);
+} else {
+    $previousReleaseFilesFolders = readZipFile($options['from'], $excludedFolders);
 }
 
-$filesDifference = array_diff($previousReleaseFiles, $newReleaseFiles);
+if (is_dir($options['to'])) {
+    $newReleaseFilesFolders = readFolder($options['to'], $excludedFolders);
+} else {
+    $newReleaseFilesFolders = readZipFile($options['to'], $excludedFolders);
+}
 
-$foldersDifference = array_diff($previousReleaseFolders, $newReleaseFolders);
+$filesDifferenceAdd      = array_diff($newReleaseFilesFolders->files, $previousReleaseFilesFolders->files);
+$filesDifferenceDelete   = array_diff($previousReleaseFilesFolders->files, $newReleaseFilesFolders->files);
+$foldersDifferenceAdd    = array_diff($newReleaseFilesFolders->folders, $previousReleaseFilesFolders->folders);
+$foldersDifferenceDelete = array_diff($previousReleaseFilesFolders->folders, $newReleaseFilesFolders->folders);
 
 // Specific files (e.g. language files) that we want to keep on upgrade
 $filesToKeep = [
@@ -156,33 +244,33 @@ function usage($command)
 
 // Remove folders from the results which we want to keep on upgrade
 foreach ($foldersToKeep as $folder) {
-    if (($key = array_search($folder, $foldersDifference)) !== false) {
-        unset($foldersDifference[$key]);
+    if (($key = array_search($folder, $foldersDifferenceDelete)) !== false) {
+        unset($foldersDifferenceDelete[$key]);
     }
 }
 
-asort($filesDifference);
-rsort($foldersDifference);
+asort($filesDifferenceDelete);
+rsort($foldersDifferenceDelete);
 
 $deletedFiles = [];
 $renamedFiles = [];
 
-foreach ($filesDifference as $file) {
+foreach ($filesDifferenceDelete as $file) {
     // Don't remove any specific files (e.g. language files) that we want to keep on upgrade
     if (array_search($file, $filesToKeep) !== false) {
         continue;
     }
 
     // Check for files which might have been renamed only
-    $matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFiles);
+    $matches = preg_grep('/^' . preg_quote($file, '/') . '$/i', $newReleaseFilesFolders->files);
 
     if ($matches !== false) {
         foreach ($matches as $match) {
             if (\dirname($match) === \dirname($file) && strtolower(basename($match)) === strtolower(basename($file))) {
                 // File has been renamed only: Add to renamed files list
                 $renamedFiles[] = substr($file, 0, -1) . ' => ' . $match;
 
-                // Go on with the next file in $filesDifference
+                // Go on with the next file in $filesDifferenceDelete
                 continue 2;
             }
         }
@@ -193,9 +281,54 @@ function usage($command)
 }
 
 // Write the lists to files for later reference
-file_put_contents(__DIR__ . '/deleted_files.txt', implode("\n", $deletedFiles));
-file_put_contents(__DIR__ . '/deleted_folders.txt', implode("\n", $foldersDifference));
-file_put_contents(__DIR__ . '/renamed_files.txt', implode("\n", $renamedFiles));
+$addedFilesFile     = __DIR__ . '/added_files.txt';
+$addedFoldersFile   = __DIR__ . '/added_folders.txt';
+$deletedFilesFile   = __DIR__ . '/deleted_files.txt';
+$deletedFoldersFile = __DIR__ . '/deleted_folders.txt';
+$renamedFilesFile   = __DIR__ . '/renamed_files.txt';
+
+@unlink($addedFilesFile);
+@unlink($addedFoldersFile);
+@unlink($deletedFilesFile);
+@unlink($deletedFoldersFile);
+@unlink($renamedFilesFile);
+
+if (\count($filesDifferenceAdd) > 0) {
+    file_put_contents($addedFilesFile, implode("\n", $filesDifferenceAdd));
+}
+
+if (\count($foldersDifferenceAdd) > 0) {
+    file_put_contents($addedFoldersFile, implode("\n", $foldersDifferenceAdd));
+}
+
+if (\count($deletedFiles) > 0) {
+    file_put_contents($deletedFilesFile, implode("\n", $deletedFiles));
+}
+
+if (\count($foldersDifferenceDelete) > 0) {
+    file_put_contents($deletedFoldersFile, implode("\n", $foldersDifferenceDelete));
+}
+
+if (\count($renamedFiles) > 0) {
+    file_put_contents($renamedFilesFile, implode("\n", $renamedFiles));
+}
+
+echo PHP_EOL;
+echo 'There are ' . PHP_EOL;
+echo ' - ' . \count($filesDifferenceAdd) . ' added files, ' . PHP_EOL;
+echo ' - ' . \count($foldersDifferenceAdd) . ' added folders, ' . PHP_EOL;
+echo ' - ' . \count($deletedFiles) . ' deleted files, ' . PHP_EOL;
+echo ' - ' . \count($foldersDifferenceDelete) .  ' deleted folders and ' . PHP_EOL;
+echo ' - ' . \count($renamedFiles) .  ' renamed files' . PHP_EOL;
+echo PHP_EOL;
+echo 'in comparison' . PHP_EOL;
+echo ' from "' . $options['from'] . '"' . PHP_EOL;
+echo ' to "' . $options['to'] . '"' . PHP_EOL;
+echo PHP_EOL;
+echo 'The following folders and their subfolders have been skipped so they were not included in the comparison:' . PHP_EOL;
+
+foreach ($excludedFolders as $excludedFolder) {
+    echo ' - ' . $excludedFolder . PHP_EOL;
+}
 
 echo PHP_EOL;
-echo 'There are ' . \count($deletedFiles) . ' deleted files, ' . \count($foldersDifference) .  ' deleted folders and ' . \count($renamedFiles) .  ' renamed files in comparison to "' . $options['from'] . '"' . PHP_EOL;