Porting zip export import to stable8.0 (#9237)

* Add button to download multiple projects as a zip file (#8936)

* Add download zip button to scriptmanager

* better zip name

* Only show zip when multiple files are selected

* Update package and thirdparty notices

* Add ability to import zip files of projects (#8938)

* Add ability to import zip files of projects

* Add tick event

Co-authored-by: Richard Knoll <riknoll@users.noreply.github.com>

package.json

@@ -64,6 +64,7 @@
     "@blockly/plugin-workspace-search": "^4.0.10",
     "@fortawesome/fontawesome-free": "^5.15.4",
     "@microsoft/immersive-reader-sdk": "1.1.0",
+    "@zip.js/zip.js": "2.4.20",
     "applicationinsights-js": "^1.0.20",
     "browserify": "16.2.3",
     "chai": "^3.5.0",

webapp/public/index.html

@@ -106,7 +106,8 @@
             "blockly.css": "/blb/blockly.css",
             "rtlblockly.css": "/blb/rtlblockly.css",
             "gifjs/gif.js": "/blb/gifjs/gif.js",
-            "qrcode/qrcode.min.js": "/blb/qrcode/qrcode.min.js"
+            "qrcode/qrcode.min.js": "/blb/qrcode/qrcode.min.js",
+            "zip.js/zip.min.js": "/blb/zip.js/zip.min.js"
         }
     </script>
 </body>

webapp/src/app.tsx

@@ -67,6 +67,8 @@ type FileHistoryEntry = pxt.editor.FileHistoryEntry;
 type EditorSettings = pxt.editor.EditorSettings;
 type ProjectCreationOptions = pxt.editor.ProjectCreationOptions;
 
+declare const zip: any;
+
 import Cloud = pxt.Cloud;
 import Util = pxt.Util;
 import { HintManager } from "./hinttooltip";
@@ -1987,6 +1989,10 @@ export class ProjectView
         return false
     }
 
+    isZipFile(filename: string): boolean {
+        return /\.zip$/i.test(filename)
+    }
+
     importProjectCoreAsync(buf: Uint8Array, options?: pxt.editor.ImportFileOptions) {
         return (buf[0] == '{'.charCodeAt(0) ?
             Promise.resolve(pxt.U.uint8ArrayToString(buf)) :
@@ -2055,6 +2061,71 @@ export class ProjectView
             .then(buf => this.importProjectCoreAsync(buf, options))
     }
 
+   async importZipFileAsync(file: File, options?: pxt.editor.ImportFileOptions) {
+        if (!file) return;
+        pxt.tickEvent("import.zip");
+
+        await scriptmanager.loadZipAsync();
+        const reader = new zip.ZipReader(new zip.BlobReader(file));
+
+        const zippedFiles = (await reader.getEntries()).filter((zipped: any) => this.isProjectFile(zipped.filename));
+        let progress = {
+            done: 0
+        }
+
+        if (zippedFiles.length === 0) {
+            core.warningNotification(lf("No projects available to import found inside zip file."));
+            return;
+        }
+
+        const confirmed = await core.confirmAsync({
+            header: lf("Import zip file?"),
+            body: lf("Do you want to import all projects in this zip file? This will import {0} projects.", zippedFiles.length),
+            agreeLbl: lf("Okay"),
+            hasCloseIcon: true,
+        });
+
+        if (!confirmed) return;
+
+        let cancelled = false;
+
+        core.dialogAsync({
+            header: lf("Extracting files..."),
+            jsxd: () => <dialogs.ProgressBar percentage={100 * (progress.done / zippedFiles.length)} />,
+            onClose: () => cancelled = true
+        })
+
+        for (const zipped of zippedFiles) {
+            try {
+                const buf: Uint8Array = await zipped.getData(new zip.Uint8ArrayWriter());
+
+                const data = JSON.parse(await pxt.lzmaDecompressAsync(buf)) as pxt.cpp.HexFile;
+
+                let h: pxt.workspace.InstallHeader = {
+                    target: pxt.appTarget.id,
+                    targetVersion: data.meta.targetVersions ? data.meta.targetVersions.target : undefined,
+                    editor: data.meta.editor,
+                    name: data.meta.name,
+                    meta: {},
+                    pubId: "",
+                    pubCurrent: false
+                }
+
+                const files = JSON.parse(data.source) as pxt.Map<string>;
+                await workspace.installAsync(h, files);
+            }
+            catch (e) {
+
+            }
+
+            if (cancelled) break;
+            progress.done ++;
+            core.forceUpdate();
+        }
+
+        core.hideDialog();
+    }
+
     importPNGBuffer(buf: ArrayBuffer) {
         pxt.Util.decodeBlobAsync("data:image/png;base64," +
             btoa(pxt.Util.uint8ArrayToString(new Uint8Array(buf))))
@@ -2224,7 +2295,7 @@ export class ProjectView
 
     initDragAndDrop() {
         draganddrop.setupDragAndDrop(document.body,
-            file => file.size < 1000000 && this.isHexFile(file.name) || this.isBlocksFile(file.name),
+            file => file.size < 1000000 && this.isHexFile(file.name) || this.isBlocksFile(file.name) || this.isZipFile(file.name),
             files => {
                 if (files) {
                     pxt.tickEvent("dragandrop.open")
@@ -2267,6 +2338,8 @@ export class ProjectView
             this.importAssetFile(file)
         } else if (this.isPNGFile(file.name)) {
             this.importPNGFile(file, options);
+        } else if (this.isZipFile(file.name)) {
+            this.importZipFileAsync(file, options);
         } else {
             const importer = this.resourceImporters.filter(fi => fi.canImport(file))[0];
             if (importer) {

webapp/src/dialogs.tsx

@@ -305,7 +305,7 @@ interface ProgressBarProps {
     cornerRadius?: number;
 }
 
-class ProgressBar extends React.Component<ProgressBarProps, {}> {
+export class ProgressBar extends React.Component<ProgressBarProps, {}> {
     render() {
         let { percentage, label, cornerRadius } = this.props;
 

webapp/src/scriptmanager.tsx

@@ -9,6 +9,19 @@ import * as auth from "./auth";
 import { SearchInput } from "./components/searchInput";
 import { ProjectsCodeCard } from "./projects";
 import { fireClickOnEnter } from "./util";
+import { Modal } from "../../react-common/components/controls/Modal";
+import { ProgressBar } from "./dialogs";
+
+declare const zip: any;
+
+let loadZipJsPromise: Promise<boolean>;
+export function loadZipAsync(): Promise<boolean> {
+    if (!loadZipJsPromise)
+        loadZipJsPromise = pxt.BrowserUtils.loadScriptAsync("zip.js/zip.min.js")
+            .then(() => typeof zip !== "undefined")
+            .catch(e => false)
+    return loadZipJsPromise;
+}
 
 type ISettingsProps = pxt.editor.ISettingsProps;
 
@@ -30,6 +43,13 @@ export interface ScriptManagerDialogState {
 
     sortedBy?: string;
     sortedAsc?: boolean;
+
+    download?: DownloadProgress;
+}
+
+interface DownloadProgress {
+    completed: number;
+    max: number;
 }
 
 export class ScriptManagerDialog extends data.Component<ScriptManagerDialogProps, ScriptManagerDialogState> {
@@ -280,6 +300,125 @@ export class ScriptManagerDialog extends data.Component<ScriptManagerDialogProps
         this.setState({ sortedAsc: !sortedAsc });
     }
 
+    handleDownloadAsync = async () => {
+        pxt.tickEvent("scriptmanager.downloadZip", undefined, { interactiveConsent: true });
+
+        await loadZipAsync();
+
+        const { selected } = this.state;
+        const zipWriter = new zip.ZipWriter(new zip.Data64URIWriter("application/zip"));
+        const selectedHeaders = this.getSortedHeaders().filter(h => selected[this.getId(h)]);
+
+        let done = 0;
+
+        const takenNames: {[index: string]: boolean} = {};
+
+        const format = (val: number, len = 2) => {
+            let out = val + "";
+            while (out.length < len) {
+                out = "0" + out
+            }
+
+            return out.substring(0, len);
+        }
+
+        this.setState({
+            download: {
+                completed: 0,
+                max: selectedHeaders.length
+            }
+        });
+
+        const targetNickname = pxt.appTarget.nickname || pxt.appTarget.id;
+
+        for (const header of selectedHeaders) {
+            const text = await workspace.getTextAsync(header.id);
+
+            let preferredEditor = "blocksprj";
+            try {
+                const config = JSON.parse(text["pxt.json"]) as pxt.PackageConfig;
+
+                preferredEditor = config.preferredEditor || "blocksprj"
+            }
+            catch (e) {
+                // ignore invalid configs
+            }
+
+            const project: pxt.cpp.HexFile = {
+                meta: {
+                    cloudId: pxt.CLOUD_ID + pxt.appTarget.id,
+                    targetVersions: pxt.appTarget.versions,
+                    editor: preferredEditor,
+                    name: header.name
+                },
+                source: JSON.stringify(text, null, 2)
+            };
+
+            const compressed = await pxt.lzmaCompressAsync(JSON.stringify(project, null, 2));
+
+            /* eslint-disable no-control-regex */
+            let sanitizedName = header.name.replace(/[()\\\/.,?*^:<>!;'#$%^&|"@+=«»°{}\[\]¾½¼³²¦¬¤¢£~­¯¸`±\x00-\x1F]/g, '');
+            sanitizedName = sanitizedName.trim().replace(/\s+/g, '-');
+            /* eslint-enable no-control-regex */
+
+            if (pxt.appTarget.appTheme && pxt.appTarget.appTheme.fileNameExclusiveFilter) {
+                const rx = new RegExp(pxt.appTarget.appTheme.fileNameExclusiveFilter, 'g');
+                sanitizedName = sanitizedName.replace(rx, '');
+            }
+
+            if (!sanitizedName) {
+                sanitizedName = "Untitled"; // do not translate to avoid unicode issues
+            }
+
+            // Include the recent use time in the filename
+            const date = new Date(header.recentUse * 1000);
+            const dateSnippet = `${date.getFullYear()}-${format(date.getMonth())}-${format(date.getDate())}`
+
+            // FIXME: handle different date formatting?
+            let fn = `${targetNickname}-${dateSnippet}-${sanitizedName}.mkcd`;
+
+            // zip.js can't handle multiple files with the same name
+            if (takenNames[fn]) {
+                let index = 2;
+                do {
+                    fn = `${targetNickname}-${dateSnippet}-${sanitizedName}${index}.mkcd`
+                    index ++;
+                } while(takenNames[fn])
+            }
+
+            takenNames[fn] = true;
+
+            await zipWriter.add(fn, new zip.Uint8ArrayReader(compressed));
+
+            // Check for cancellation
+            if (!this.state.download) return;
+
+            done++;
+            this.setState({
+                download: {
+                    completed: done,
+                    max: selectedHeaders.length
+                }
+            });
+        }
+
+        const datauri = await zipWriter.close();
+
+        const zipName = `makecode-${targetNickname}-project-download.zip`
+
+        pxt.BrowserUtils.browserDownloadDataUri(datauri, zipName);
+
+        this.setState({
+            download: null
+        });
+    }
+
+    handleDownloadProgressClose = () => {
+        this.setState({
+            download: null
+        });
+    }
+
     private getSelectedHeader() {
         const { selected } = this.state;
         const indexes = Object.keys(selected);
@@ -315,7 +454,7 @@ export class ScriptManagerDialog extends data.Component<ScriptManagerDialogProps
     }
 
     renderCore() {
-        const { visible, selected, markedNew, view, searchFor, sortedBy, sortedAsc } = this.state;
+        const { visible, selected, markedNew, view, searchFor, sortedBy, sortedAsc, download } = this.state;
         if (!visible) return <div></div>;
 
         const darkTheme = pxt.appTarget.appTheme.baseTheme == 'dark';
@@ -344,8 +483,10 @@ export class ScriptManagerDialog extends data.Component<ScriptManagerDialogProps
                 style={{ flexGrow: 1 }}
                 searchOnChange={true}
             />);
-            if (Object.keys(selected).length > 0) {
-                if (Object.keys(selected).length == 1) {
+
+            const numSelected = Object.keys(selected).length
+            if (numSelected > 0) {
+                if (numSelected == 1) {
                     const openBtn = <sui.Button key="edit" icon="edit outline" className="icon"
                         text={lf("Open")} textClass="landscape only" title={lf("Open Project")} onClick={this.handleOpen} />;
                     if (!openNewTab)
@@ -363,6 +504,10 @@ export class ScriptManagerDialog extends data.Component<ScriptManagerDialogProps
                 }
                 headerActions.push(<sui.Button key="delete" icon="trash" className="icon red"
                     text={lf("Delete")} textClass="landscape only" title={lf("Delete Project")} onClick={this.handleDelete} />);
+                if (numSelected > 1) {
+                    headerActions.push(<sui.Button key="download-zip" icon="download" className="icon"
+                        text={lf("Download Zip")} textClass="landscape only" title={lf("Download Zip")} onClick={this.handleDownloadAsync} />);
+                }
                 headerActions.push(<div key="divider" className="divider"></div>);
             }
             headerActions.push(<sui.Button key="view" icon={view == 'grid' ? 'th list' : 'grid layout'} className="icon"
@@ -465,6 +610,11 @@ export class ScriptManagerDialog extends data.Component<ScriptManagerDialogProps
                         </table>
                     </div>
                     : undefined}
+
+                {download &&
+                    <Modal title={lf("Preparing your zip file...")} onClose={this.handleDownloadProgressClose}>
+                        <ProgressBar percentage={100 * (download.completed / download.max)} />
+                    </Modal>}
             </sui.Modal>
         )
     }