Download all projects from a discourse tag (microsoft#6570)

pelikhan · jwunderl · web-flow · commit 4bc221381013 · 2020-02-20T07:38:24.000-08:00
* moving discourse to pxtlib

* add cli to download topics from tag

* expose more types

* download and regen markdown

* fix check

* error checking

* don't download script twice

* simplify discourse interface

* handle cli

* handle ill formatted post

* cli type

* cli tweaks

* logging

* apply replacement

* fix regex

* cleanup

* fix build

* junk file

* store forum images locally

* pul thumb for makecode

* drop screenshots on disk

* download from forum image

* don't sort

* add "gulp cli" support

* don't rewrite existing scripts

* extract rx

* Update cli/cli.ts

Co-Authored-By: Joey Wunderlich &lt;jwunderl@users.noreply.github.com&gt;

* revert fix

* undo changes

* fix build

* clean out id

* clean out id

Co-authored-by: Joey Wunderlich &lt;jwunderl@users.noreply.github.com&gt;
diff --git a/cli/cli.ts b/cli/cli.ts
@@ -3130,6 +3130,116 @@ export function exportCppAsync(parsed: commandParser.ParsedCommand) {
         })
 }
 
+export function downloadDiscourseTagAsync(parsed: commandParser.ParsedCommand): Promise<void> {
+    const rx = /```codecard((.|\s)*)```/;
+    const tag = parsed.args[0] as string;
+    if (!tag)
+        U.userError("Missing tag")
+    const out = parsed.flags["out"] as string || "temp";
+    const outmd = parsed.flags["md"] as string;
+    const discourseRoot = pxt.appTarget.appTheme
+        && pxt.appTarget.appTheme.socialOptions
+        && pxt.appTarget.appTheme.socialOptions.discourse;
+    if (!discourseRoot)
+        U.userError("Target not configured for discourse");
+    if (outmd && !fs.existsSync(outmd))
+        U.userError(`${outmd} file not found`)
+    let md: string = outmd && fs.readFileSync(outmd, { encoding: "utf8" });
+
+    nodeutil.mkdirP(out);
+    let n = 0;
+    let cards: pxt.CodeCard[] = [];
+    // parse existing cards
+    if (md) {
+        md.replace(rx, (m, c) => {
+            cards = JSON.parse(c);
+            return "";
+        })
+    }
+    return pxt.discourse.topicsByTag(discourseRoot, tag)
+        .then(topics => Promise.mapSeries(topics, topic => {
+            pxt.log(`  ${topic.title}`)
+            return pxt.discourse.extractSharedIdFromPostUrl(topic.url)
+                .then(id => {
+                    if (!id) {
+                        pxt.log(`  --> unknown project id`)
+                        return Promise.resolve();
+                    }
+                    n++;
+                    return extractAsyncInternal(id, out, false)
+                        .then(() => {
+                            // does the current card have an image?
+                            let card = cards.filter(c => c.url == topic.url)[0];
+                            if (card && card.imageUrl) {
+                                pxt.log(`${card.name} already in markdown`)
+                                return Promise.resolve(); // already handled
+                            }
+                            // new card? add to list
+                            if (!card) {
+                                card = topic;
+                                card.description = "";
+                                cards.push(card);
+                            }
+
+                            const pfn = `./docs/static/discourse/${id}.`;
+                            if (md && !["png", "jpg", "gif"].some(ext => nodeutil.fileExistsSync(pfn + ext))) {
+                                return downloadImageAsync(id, topic, `https://makecode.com/api/${id}/thumb`)
+                                    .catch(e => {
+                                        // no image
+                                        pxt.debug(`no thumb ${e}`);
+                                        // use image from forum
+                                        if (topic.imageUrl)
+                                            return downloadImageAsync(id, topic, topic.imageUrl);
+                                        else
+                                            throw e; // bail out
+                                    })
+                            }
+                            return Promise.resolve();
+                        }).catch(e => {
+                            pxt.log(`error: project ${id} could not be loaded or no image`);
+                        });
+                })
+        }))
+        .then(() => {
+            if (md) {
+                // inject updated cards
+                cards.forEach(card => delete (card as any).id);
+                md = md.replace(rx, (m, c) => {
+                    return `\`\`\`codecard
+${JSON.stringify(cards, null, 4)}
+\`\`\``;
+                })
+                fs.writeFileSync(outmd, md, { encoding: "utf8" });
+            }
+            pxt.log(`downloaded ${n} programs from tag ${tag}`)
+        })
+
+    function downloadImageAsync(id: string, topic: pxt.CodeCard, url: string): Promise<void> {
+        return pxt.Util.requestAsync({
+            url: `https://makecode.com/api/${id}/thumb`,
+            method: "GET",
+            responseArrayBuffer: true,
+            headers: {
+                "accept": "image/*"
+            }
+        }).then(resp => {
+            if (resp.buffer) {
+                const m = /image\/(png|jpeg|gif)/.exec(resp.headers["content-type"] as string);
+                if (!m) {
+                    pxt.log(`unknown image type: ${resp.headers["content-type"]}`);
+                } else {
+                    let ext = m[1];
+                    if (ext == "jpeg") ext = "jpg";
+                    const ifn = `/static/discourse/${id}.${ext}`;
+                    const localifn = "./docs" + ifn;
+                    topic.imageUrl = ifn;
+                    nodeutil.writeFileSync(localifn, new Buffer(resp.buffer as ArrayBuffer));
+                }
+            }
+        });
+    }
+}
+
 export function formatAsync(parsed: commandParser.ParsedCommand) {
     let inPlace = !!parsed.flags["i"];
     let testMode = !!parsed.flags["t"];
@@ -4682,7 +4792,8 @@ export function extractAsync(parsed: commandParser.ParsedCommand): Promise<void>
     const vscode = !!parsed.flags["code"];
     const out = parsed.flags["code"] || '.';
     const filename = parsed.args[0];
-    return extractAsyncInternal(filename, out as string, vscode);
+    return extractAsyncInternal(filename, out as string, vscode)
+        .then(() => { });
 }
 
 function isScriptId(id: string) {
@@ -4722,13 +4833,13 @@ function fetchTextAsync(filename: string): Promise<Buffer> {
         return readFileAsync(filename)
 }
 
-function extractAsyncInternal(filename: string, out: string, vscode: boolean): Promise<void> {
+function extractAsyncInternal(filename: string, out: string, vscode: boolean): Promise<string[]> {
     if (filename && nodeutil.existsDirSync(filename)) {
         pxt.log(`extracting folder ${filename}`);
         return Promise.all(fs.readdirSync(filename)
             .filter(f => /\.(hex|uf2)/.test(f))
             .map(f => extractAsyncInternal(path.join(filename, f), out, vscode)))
-            .then(() => { });
+            .then(() => [filename]);
     }
 
     return fetchTextAsync(filename)
@@ -4738,6 +4849,7 @@ function extractAsyncInternal(filename: string, out: string, vscode: boolean): P
                 pxt.debug('launching code...')
                 dirs.forEach(dir => openVsCode(dir));
             }
+            return dirs;
         })
 }
 
@@ -4777,31 +4889,31 @@ function extractBufferAsync(buf: Buffer, outDir: string): Promise<string[]> {
         .then(() => {
             let str = buf.toString("utf8")
             if (str[0] == ":") {
-                console.log("Detected .hex file.")
+                pxt.debug("Detected .hex file.")
                 return unpackHexAsync(buf)
             } else if (str[0] == "U") {
-                console.log("Detected .uf2 file.")
+                pxt.debug("Detected .uf2 file.")
                 return unpackHexAsync(buf)
             } else if (str[0] == "{") {  // JSON
-                console.log("Detected .json file.")
+                pxt.debug("Detected .json file.")
                 return JSON.parse(str)
             } else if (buf[0] == 0x5d) { // JSZ
-                console.log("Detected .jsz/.pxt file.")
+                pxt.debug("Detected .jsz/.pxt file.")
                 return pxt.lzmaDecompressAsync(buf as any)
                     .then(str => JSON.parse(str))
             } else
                 return Promise.resolve(null)
         })
         .then(json => {
             if (!json) {
-                console.log("Couldn't extract.")
+                pxt.log("Couldn't extract.")
                 return undefined;
             }
             if (json.meta && json.source) {
                 json = typeof json.source == "string" ? JSON.parse(json.source) : json.source
             }
             if (Array.isArray(json.scripts)) {
-                console.log("Legacy TD workspace.")
+                pxt.debug("Legacy TD workspace.")
                 json.projects = json.scripts.map((scr: any) => ({
                     name: scr.header.name,
                     files: oneFile(scr.source, scr.header.editor)
@@ -4810,8 +4922,8 @@ function extractBufferAsync(buf: Buffer, outDir: string): Promise<string[]> {
             }
 
             if (json[pxt.CONFIG_NAME]) {
-                console.log("Raw JSON files.")
-                let cfg: pxt.PackageConfig = JSON.parse(json[pxt.CONFIG_NAME])
+                pxt.debug("Raw JSON files.")
+                let cfg: pxt.PackageConfig = pxt.Package.parseAndValidConfig(json[pxt.CONFIG_NAME])
                 let files = json
                 json = {
                     projects: [{
@@ -4823,7 +4935,7 @@ function extractBufferAsync(buf: Buffer, outDir: string): Promise<string[]> {
 
             let prjs: SavedProject[] = json.projects
             if (!prjs) {
-                console.log("No projects found.")
+                pxt.log("No projects found.")
                 return undefined;
             }
             const dirs = writeProjects(prjs, outDir)
@@ -6148,6 +6260,26 @@ ${pxt.crowdin.KEY_VARIABLE} - crowdin key
         argString: "<target-directory>"
     }, exportCppAsync);
 
+    p.defineCommand({
+        name: "downloaddiscoursetag",
+        aliases: ["ddt"],
+        help: "Download program for a discourse tag",
+        advanced: true,
+        argString: "<tag>",
+        flags: {
+            out: {
+                description: "output folder, default is temp",
+                argument: "out",
+                type: "string"
+            },
+            md: {
+                description: "path of the markdown file to generate",
+                argument: "out",
+                type: "string"
+            }
+        }
+    }, downloadDiscourseTagAsync)
+
     function simpleCmd(name: string, help: string, callback: (c?: commandParser.ParsedCommand) => Promise<void>, argString?: string, onlineHelp?: boolean): void {
         p.defineCommand({ name, help, onlineHelp, argString }, callback);
     }
diff --git a/gulpfile.js b/gulpfile.js
@@ -612,4 +612,4 @@ exports.karma = karma;
 exports.update = update;
 exports.uglify = runUglify;
 exports.watch = initWatch;
-exports.watchCli = initWatchCli;
+exports.watchCli = initWatchCli;
diff --git a/pxtlib/discourse.ts b/pxtlib/discourse.ts
@@ -0,0 +1,72 @@
+namespace pxt.discourse {
+    interface DiscoursePostResponse {
+        featured_link?: string;
+        post_stream?: DiscoursePostStream;
+    }
+    interface DiscoursePostStream {
+        posts?: DiscoursePost[];
+    }
+    interface DiscoursePost {
+        link_counts?: DiscourseLinkCount[];
+    }
+    interface DiscourseLinkCount {
+        url?: string;
+    }
+    interface TagTopic {
+        id: string;
+        title: string;
+        image_url: string;
+        slug: string;
+        views: number;
+        like_count: number;
+        posters: {
+            user_id: string;
+        }[];
+    }
+    interface TagUser {
+        id: number;
+        username: string;
+        name: string;
+        avatar_template: string;
+    }
+    interface TagsResponse {
+        users: TagUser[];
+        topic_list: {
+            topics: TagTopic[];
+        }
+    }
+
+    export function extractSharedIdFromPostUrl(url: string): Promise<string> {
+        // https://docs.discourse.org/#tag/Posts%2Fpaths%2F~1posts~1%7Bid%7D.json%2Fget
+        return pxt.Util.httpGetJsonAsync(url + ".json")
+            .then((json: DiscoursePostResponse) => {
+                // extract from post_stream
+                let projectId = json.post_stream
+                    && json.post_stream.posts
+                    && json.post_stream.posts[0]
+                    && json.post_stream.posts[0].link_counts
+                        .map(link => pxt.Cloud.parseScriptId(link.url))
+                        .filter(id => !!id)[0];
+                return projectId;
+            });
+    }
+
+    export function topicsByTag(apiUrl: string, tag: string): Promise<pxt.CodeCard[]> {
+        apiUrl = apiUrl.replace(/\/$/, '');
+        const q = `${apiUrl}/tags/${tag}.json`;
+        return pxt.Util.httpGetJsonAsync(q)
+            .then((json: TagsResponse) => {
+                const users = pxt.Util.toDictionary(json.users, u => u.id.toString());
+                return json.topic_list.topics.map(t => {
+                    return <pxt.CodeCard>{
+                        id: t.id,
+                        title: t.title,
+                        url: `${apiUrl}/t/${t.slug}/${t.id}`,
+                        imageUrl: t.image_url,
+                        author: users[t.posters[0].user_id].name,
+                        cardType: "forumUrl"
+                    }
+                });
+            });
+    }
+}
diff --git a/pxtlib/initprj.ts b/pxtlib/initprj.ts
@@ -230,14 +230,13 @@ jobs:
         const configMap = JSON.parse(files[pxt.CONFIG_NAME])
         if (options)
             Util.jsonMergeFrom(configMap, options);
-        if (pxt.webConfig)
-            Object.keys(pxt.webConfig)
-                .forEach(k => configMap[k.toLowerCase()] = (<any>pxt.webConfig)[k]);
-        configMap["platform"] = pxt.appTarget.platformid || pxt.appTarget.id
-        configMap["target"] = pxt.appTarget.id
-        configMap["docs"] = pxt.appTarget.appTheme.homeUrl || "./";
-        configMap["homeurl"] = pxt.appTarget.appTheme.homeUrl || "???";
-
+        if (pxt.webConfig) { // CLI
+            Object.keys(pxt.webConfig).forEach(k => configMap[k.toLowerCase()] = (<any>pxt.webConfig)[k]);
+            configMap["platform"] = pxt.appTarget.platformid || pxt.appTarget.id
+            configMap["target"] = pxt.appTarget.id
+            configMap["docs"] = pxt.appTarget.appTheme.homeUrl || "./";
+            configMap["homeurl"] = pxt.appTarget.appTheme.homeUrl || "???";
+        }
         U.iterMap(files, (k, v) => {
             v = v.replace(/@([A-Z]+)@/g, (f, n) => configMap[n.toLowerCase()] || "")
             files[k] = v
diff --git a/webapp/src/discourse.ts b/webapp/src/discourse.ts
diff --git a/webapp/src/projects.tsx b/webapp/src/projects.tsx
@@ -8,7 +8,6 @@ import * as core from "./core";
 import * as cloud from "./cloud";
 import * as cloudsync from "./cloudsync";
 
-import * as discourse from "./discourse";
 import * as codecard from "./codecard"
 import * as carousel from "./carousel";
 import { showAboutDialogAsync } from "./dialogs";
@@ -830,7 +829,7 @@ export class ProjectsDetail extends data.Component<ProjectsDetailProps, Projects
 
     handleOpenForumUrlInEditor() {
         const { url } = this.props;
-        discourse.extractSharedIdFromPostUrl(url)
+        pxt.discourse.extractSharedIdFromPostUrl(url)
             .then(projectId => {
                 // if we have a projectid, load it
                 if (projectId)
