draft version of program output with docusaurus

.gitignore

@@ -90,3 +90,4 @@ env
 .history
 newdocs/.docusaurus
 newdocs/docs/variables.json
+newdocs/docs/sources/

Makefile

@@ -82,7 +82,7 @@ test: clean
 	OMP_NUM_THREADS=1 poetry run pytest tests -n $(JOBS) --cov rasa
 
 docs:
-	cd newdocs/ && poetry run yarn variables && yarn build
+	cd newdocs/ && poetry run yarn pre-build && yarn build
 
 livedocs:
 	cd newdocs/ && poetry run yarn start

newdocs/compile_program_outputs.js

@@ -0,0 +1,10 @@
+const getProgramOutputs = require('./plugins/program_output.js');
+
+
+
+console.info('Computing program outputs');
+getProgramOutputs({
+    docsDir: './docs',
+    include: ['**.mdx', '**.md'],
+    commandPrefix: 'poetry run',
+});

newdocs/docs/api/rasa-sdk.mdx

@@ -54,39 +54,7 @@ manually specify an actions module or package with the `--actions` flag.
 
 The full list of options for running the action server with either command is:
 
-```text
-usage: rasa run actions [-h] [-v] [-vv] [--quiet] [-p PORT]
-                        [--cors [CORS [CORS ...]]] [--actions ACTIONS]
-                        [--ssl-keyfile SSL_KEYFILE]
-                        [--ssl-certificate SSL_CERTIFICATE]
-                        [--ssl-password SSL_PASSWORD] [--auto-reload]
-
-optional arguments:
-  -h, --help            show this help message and exit
-  -p PORT, --port PORT  port to run the server at (default: 5055)
-  --cors [CORS [CORS ...]]
-                        enable CORS for the passed origin. Use * to whitelist
-                        all origins (default: None)
-  --actions ACTIONS     name of action package to be loaded (default: None)
-  --ssl-keyfile SSL_KEYFILE
-                        Set the SSL certificate to create a TLS secured
-                        server. (default: None)
-  --ssl-certificate SSL_CERTIFICATE
-                        Set the SSL certificate to create a TLS secured
-                        server. (default: None)
-  --ssl-password SSL_PASSWORD
-                        If your ssl-keyfile is protected by a password, you
-                        can specify it using this paramer. (default: None)
-  --auto-reload         Enable auto-reloading of modules containing Action
-                        subclasses. (default: False)
-
-Python Logging Options:
-  -v, --verbose         Be verbose. Sets logging level to INFO. (default:
-                        None)
-  -vv, --debug          Print lots of debugging statements. Sets logging level
-                        to DEBUG. (default: None)
-  --quiet               Be quiet! Sets logging level to WARNING. (default:
-                        None)
+```text [rasa run actions --help]
 ```
 
 ## Actions

newdocs/docusaurus.config.js

@@ -1,5 +1,7 @@
 const remarkSources = require('remark-sources');
 const remarkCollapse = require('remark-collapse');
+const { remarkProgramOutput } = require("./plugins/program_output");
+
 let versions = [];
 try {
   versions = require('./versions.json');
@@ -118,7 +120,7 @@ module.exports = {
           routeBasePath: '/',
           sidebarPath: require.resolve('./sidebars.js'),
           editUrl: 'https://github.com/rasahq/rasa/edit/master/docs/',
-          remarkPlugins: [[remarkCollapse, { test: '' }], remarkSources],
+          remarkPlugins: [[remarkCollapse, { test: '' }], remarkSources, remarkProgramOutput],
         },
         theme: {
           customCss: require.resolve('./src/css/custom.css'),

newdocs/package.json

@@ -3,10 +3,12 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "start": "yarn run variables && docusaurus start",
+    "start": "yarn run pre-build && docusaurus start",
+    "pre-build": "yarn run variables && yarn run program-outputs",
     "build": "docusaurus build",
     "new-version": "docusaurus docs:version",
-    "variables": "node compile_variables.js"
+    "variables": "node compile_variables.js",
+    "program-outputs": "node compile_program_outputs.js"
   },
   "dependencies": {
     "@docusaurus/core": "^2.0.0-alpha.56",

newdocs/plugins/program_output.js

@@ -0,0 +1,92 @@
+const fs = require('fs');
+const globby = require('globby');
+const visitChildren = require('unist-util-visit-children');
+const { promisify } = require('util');
+
+const exec = promisify(require('child_process').exec);
+const { readFile, writeFile } = fs.promises;
+
+
+const PROGRAM_OUTPUT_RE = /```[a-z]+ \[([^\]]+)\]\n```/;
+
+const defaultOptions = {
+    docsDir: './docs',
+    sourceDir: './docs/sources',
+    include: ['**.mdx', '**.md'],
+    commandPrefix: '',
+};
+
+async function getProgramOutputs(options) {
+
+    options = { ...defaultOptions, ...options };
+    const { docsDir, include, sourceDir, commandPrefix } = options;
+    const docsFiles = await globby(include, {
+      cwd: docsDir,
+    });
+    const seen = new Set();
+    let commands = await Promise.all(docsFiles.map(async (source) => {
+        const data = await readFile(`${docsDir}/${source}`);
+        const commands = [];
+        let group, command, stdout;
+        const re = new RegExp(PROGRAM_OUTPUT_RE, 'gi');
+        while ((group = re.exec(data)) !== null) {
+            command = group[1];
+            if (seen.has(command)) {
+                continue;
+            }
+            seen.add(command);
+            output = await exec(`${commandPrefix} ${command}`);
+            commands.push([command, output.stdout]);
+        }
+        return commands;
+    }));
+    commands = commands.flat().filter(pair => pair.length > 0);
+
+    return await Promise.all(commands.map(async ([command, output]) => {
+        return await writeFile(`${sourceDir}/${commandToFilename(command)}`, output);
+    }));
+};
+
+
+function remarkProgramOutput(options = {}) {
+    options = { ...defaultOptions, ...options };
+    return (root) => {
+        visitChildren((node, index, parent) => {
+            // console.info("node.type", node.type);
+            if (node && node.type === 'code') {
+                const content = readCommandOutput(node.meta, options);
+                console.info('CONTENT', content);
+                if (content !== undefined) {
+                    console.info('assigning value', content);
+                    node.value = content;
+                }
+            }
+        })(root);
+    };
+}
+
+
+function readCommandOutput(meta, { sourceDir }) {
+    if (!meta) {
+        return undefined;
+    }
+    if (meta[0] !== '[' || meta[meta.length - 1] !== ']') {
+        return undefined;
+    }
+    meta = meta.slice(1, -1);
+    console.info("META", meta, sourceDir, commandToFilename(meta));
+    try {
+        return fs.readFileSync(`${sourceDir}/${commandToFilename(meta)}`, { encoding: 'utf8' });
+    } catch (e) {
+        throw new Error(`Failed to read file: ${meta}`);
+    }
+}
+
+
+function commandToFilename(command) {
+    return command.replace(/[^a-z0-9\-]/gi, '_').toLowerCase() + '.txt';
+}
+
+
+module.exports = getProgramOutputs;
+module.exports.remarkProgramOutput = remarkProgramOutput;

scripts/push_docs_to_branch.sh

@@ -8,6 +8,7 @@ PATTERN_FOR_NEW_VERSION="^refs/tags/[0-9]+\\.[0-9]+\\.0$"
 PATTERN_FOR_PATCH_VERSION="^refs/tags/[0-9]+\\.[0-9]+\\.[1-9]+$"
 MASTER_REF=refs/heads/master
 VARIABLES_JSON=docs/docs/variables.json
+SOURCES_FILES=docs/docs/sources/
 
 [[ ! $GITHUB_REF =~ $PATTERN_FOR_NEW_VERSION ]] \
 && [[ ! $GITHUB_REF =~ $PATTERN_FOR_PATCH_VERSION ]] \
@@ -65,7 +66,7 @@ then
 else
     echo "Pushing changes to git..."
     git add .
-    git add --force $VARIABLES_JSON
+    git add --force $VARIABLES_JSON $SOURCES_FILES
     git commit -am "AUTO docusaurus $TODAY"
     git fetch --unshallow
     git push origin $DOCS_BRANCH