Skip to content

Commit

Permalink
Improve support for bulk loading tiddlers under Node.js
Browse files Browse the repository at this point in the history 
Fixes TiddlyWiki#2610
  • Loading branch information
@Jermolene
Jermolene committed Oct 15, 2016
1 parent d7b6917 commit 1b41b44
Show file tree
Hide file tree
Showing 10 changed files with 344 additions and 364 deletions.
152 changes: 108 additions & 44 deletions boot/boot.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1456,17 +1456,26 @@ $tw.loadTiddlersFromFile = function(filepath,fields) {
typeInfo = type ? $tw.config.contentTypeInfo[type] : null,
data = fs.readFileSync(filepath,typeInfo ? typeInfo.encoding : "utf8"),
tiddlers = $tw.wiki.deserializeTiddlers(ext,data,fields),
metafile = filepath + ".meta",
metadata;
if(ext !== ".json" && tiddlers.length === 1 && fs.existsSync(metafile)) {
metadata = fs.readFileSync(metafile,"utf8");
if(metadata) {
tiddlers = [$tw.utils.parseFields(metadata,tiddlers[0])];
}
if(ext !== ".json" && tiddlers.length === 1) {
metadata = $tw.loadMetadataForFile(filepath);
tiddlers = [$tw.utils.extend({},metadata,tiddlers[0])];
}
return {filepath: filepath, type: type, tiddlers: tiddlers, hasMetaFile: !!metadata};
};

/*
Load the metadata fields in the .meta file corresponding to a particular file
*/
$tw.loadMetadataForFile = function(filepath) {
var metafilename = filepath + ".meta";
if(fs.existsSync(metafilename)) {
return $tw.utils.parseFields(fs.readFileSync(metafilename,"utf8") || "");
} else {
return null;
}
};

/*
A default set of files for TiddlyWiki to ignore during load.
This matches what NPM ignores, and adds "*.meta" to ignore tiddler
Expand All @@ -1486,44 +1495,7 @@ $tw.loadTiddlersFromPath = function(filepath,excludeRegExp) {
var files = fs.readdirSync(filepath);
// Look for a tiddlywiki.files file
if(files.indexOf("tiddlywiki.files") !== -1) {
// If so, process the files it describes
var filesInfo = JSON.parse(fs.readFileSync(filepath + path.sep + "tiddlywiki.files","utf8"));
// First the tiddlers
$tw.utils.each(filesInfo.tiddlers,function(tidInfo) {
var type = tidInfo.fields.type || "text/plain",
typeInfo = $tw.config.contentTypeInfo[type],
pathname = path.resolve(filepath,tidInfo.file),
text = fs.readFileSync(pathname,typeInfo ? typeInfo.encoding : "utf8");
if(tidInfo.isTiddlerFile) {
var fileTiddlers = $tw.wiki.deserializeTiddlers(path.extname(pathname),text) || [];
$tw.utils.each(fileTiddlers,function(tiddler) {
$tw.utils.extend(tiddler,tidInfo.fields);
if(tidInfo.prefix) {
tiddler.text = tidInfo.prefix + tiddler.text;
}
if(tidInfo.suffix) {
tiddler.text = tiddler.text + tidInfo.suffix;
}
});
tiddlers.push({tiddlers: fileTiddlers});
} else {
if(tidInfo.prefix) {
text = tidInfo.prefix + text;
}
if(tidInfo.suffix) {
text = text + tidInfo.suffix;
}
tidInfo.fields.text = text;
tiddlers.push({tiddlers: [tidInfo.fields]});
}
});
// Then any recursive directories
$tw.utils.each(filesInfo.directories,function(dirPath) {
var pathname = path.resolve(filepath,dirPath);
if(fs.existsSync(pathname) && fs.statSync(pathname).isDirectory()) {
tiddlers.push.apply(tiddlers,$tw.loadTiddlersFromPath(pathname,excludeRegExp));
}
});
Array.prototype.push.apply(tiddlers,$tw.loadTiddlersFromSpecification(filepath));
} else {
// If not, read all the files in the directory
$tw.utils.each(files,function(file) {
Expand All @@ -1539,6 +1511,98 @@ $tw.loadTiddlersFromPath = function(filepath,excludeRegExp) {
return tiddlers;
};

/*
Load all the tiddlers defined by a `tiddlywiki.files` specification file
filepath: pathname of the directory containing the specification file
*/
$tw.loadTiddlersFromSpecification = function(filepath) {
var tiddlers = [];
// Read the specification
var filesInfo = JSON.parse(fs.readFileSync(filepath + path.sep + "tiddlywiki.files","utf8"));
// Helper to process a file
var processFile = function(filename,isTiddlerFile,fields) {
var extInfo = $tw.config.fileExtensionInfo[path.extname(filename)],
type = (extInfo || {}).type || fields.type || "text/plain",
typeInfo = $tw.config.contentTypeInfo[type] || {},
pathname = path.resolve(filepath,filename),
text = fs.readFileSync(pathname,typeInfo.encoding || "utf8"),
metadata = $tw.loadMetadataForFile(pathname) || {},
fileTiddlers;
if(isTiddlerFile) {
fileTiddlers = $tw.wiki.deserializeTiddlers(path.extname(pathname),text,metadata) || [];
} else {
fileTiddlers = [$tw.utils.extend({text: text},metadata)];
}
var combinedFields = $tw.utils.extend({},fields,metadata);
$tw.utils.each(fileTiddlers,function(tiddler) {
$tw.utils.each(combinedFields,function(fieldInfo,name) {
if(typeof fieldInfo === "string" || $tw.utils.isArray(fieldInfo)) {
tiddler[name] = fieldInfo;
} else {
var value = tiddler[name];
switch(fieldInfo.source) {
case "filename":
value = path.basename(filename);
break;
case "basename":
value = path.basename(filename,path.extname(filename));
break;
case "extname":
value = path.extname(filename);
break;
case "created":
value = new Date(fs.statSync(pathname).birthtime);
break;
case "modified":
value = new Date(fs.statSync(pathname).mtime);
break;
}
if(fieldInfo.prefix) {
value = fieldInfo.prefix + value;
}
if(fieldInfo.suffix) {
value = value + fieldInfo.suffix;
}
tiddler[name] = value;
}
});
});
tiddlers.push({tiddlers: fileTiddlers});
};
// Process the listed tiddlers
$tw.utils.each(filesInfo.tiddlers,function(tidInfo) {
if(tidInfo.prefix && tidInfo.suffix) {
tidInfo.fields.text = {prefix: tidInfo.prefix,suffix: tidInfo.suffix};
} else if(tidInfo.prefix) {
tidInfo.fields.text = {prefix: tidInfo.prefix};
} else if(tidInfo.suffix) {
tidInfo.fields.text = {suffix: tidInfo.suffix};
}
processFile(tidInfo.file,tidInfo.isTiddlerFile,tidInfo.fields);
});
// Process any listed directories
$tw.utils.each(filesInfo.directories,function(dirSpec) {
// Read literal directories directly
if(typeof dirSpec === "string") {
var pathname = path.resolve(filepath,dirSpec);
if(fs.existsSync(pathname) && fs.statSync(pathname).isDirectory()) {
tiddlers.push.apply(tiddlers,$tw.loadTiddlersFromPath(pathname,excludeRegExp));
}
} else {
// Process directory specifier
var dirPath = path.resolve(filepath,dirSpec.path),
files = fs.readdirSync(dirPath),
fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$");
for(var t=0; t<files.length; t++) {
if(files[t] !== "tiddlywiki.files" && fileRegExp.test(files[t])) {
processFile(dirPath + path.sep + files[t],dirSpec.isTiddlerFile,dirSpec.fields);
}
}
}
});
return tiddlers;
};

/*
Load the tiddlers from a plugin folder, and package them up into a proper JSON plugin tiddler
*/
Expand Down
116 changes: 16 additions & 100 deletions editions/tw5.com/tiddlers/concepts/TiddlyWikiFolders.tid
View file
Original file line number Diff line number Diff line change
@@ -1,124 +1,40 @@
created: 20130825214200000
modified: 20150728104424501
modified: 20161015134454785
tags: [[TiddlyWiki on Node.js]]
title: TiddlyWikiFolders
type: text/vnd.tiddlywiki

As well as traditional single file wikis, [[TiddlyWiki on Node.js]] supports wikis that are stored as a folder of individual tiddler files.

! Wiki folder files and folders
! Wiki Folder Structure

Wiki folders can contain the following files and folders:

* ''tiddlywiki.info'' - JSON file containing metadata for the wiki
* ''tiddlywiki.info'' - JSON file containing metadata for the wiki -- see [[tiddlywiki.info Files]]
* ''\tiddlers'' - folder containing tiddler files comprising the wiki
* ''\plugins'' - folder containing [[plugin folders|PluginMechanism]] to be included in the wiki
* ''\plugins'' - folder containing [[plugin folders|PluginFolders]] to be included in the wiki
* ''\languages'' - folder containing [[plugin folders|PluginFolders]] for language plugins to be included in the wiki
* ''\themes'' - folder containing [[plugin folders|PluginFolders]] for theme plugins to be included in the wiki

Only the ''tiddlywiki.info'' file is required, the ''tiddlers'' and ''plugins'' folders are optional. Any files and folders not listed above are ignored.
Only the ''tiddlywiki.info'' file is required; everything else is optional. Any files and folders not listed above are ignored.

!! Content of `tiddlywiki.info` file
!! Plugin processing

The `tiddlywiki.info` file in a wiki folder contains a JSON object comprising the following fields:

* ''plugins'' - an array of plugin names to be included in the wiki
* ''themes'' - an array of theme names to be included in the wiki
* ''languages'' - an array of language names to be included in the wiki
* ''includeWikis'' - an array of references to external wiki folders to be included in the wiki
* ''build'' - a hashmap of named build targets, each defined by an array of command tokens (see BuildCommand)
* ''config'' - an optional hashmap of configuration options (see below)

!!! ''includeWikis''

The entries in the ''includeWikis'' array can be either a string specifying the relative path to the wiki, or an object with the following fields:

* ''path'' - relative path to wiki folder
* ''read-only'' - set //true// to prevent the tiddlers in the included wiki from being modified. The modifications will be written to the directory specified in ''default-tiddler-location'', described below

!!! ''build''

Note that the build targets of included wikis are merged if a target of that name isn't defined in the current `tiddlywiki.info` file.

!!! ''config''

Configuration options include:

* ''default-tiddler-location'' - a string path to the default location for the filesystem adaptor to save new tiddlers (resolved relative to the wiki folder)

* ''retain-original-tiddler-path'' - If true, the server will generate a tiddler [[$:/config/OriginalTiddlerPaths]] containing the original file paths of each tiddler in the wiki

!!! Example

For example:
To be usable in the browser, plugins just need to be included in the wiki. For wikis that are generated on the server, TiddlyWikiFolders can contain a [[tiddlywiki.info file|tiddlywiki.info Files]] that identifies the plugins to be included in this wiki:

```
{
"plugins": [
"tiddlywiki/tiddlyweb",
"tiddlywiki/filesystem"
],
"includeWikis": [
"../tw5.com"
],
"build": {
"index": [
"--rendertiddler","$:/core/save/all","index.html","text/plain"],
"favicon": [
"--savetiddler","$:/favicon.ico","favicon.ico",
"--savetiddler","$:/green_favicon.ico","static/favicon.ico"]
},
"config": {
"retain-original-tiddler-path": true
}
}
```

!! Content of `tiddlers` folder

All the TiddlerFiles in the `tiddlers` folder are read into the wiki at startup. Sub-folders are scanned recursively for TiddlerFiles. Newly created tiddlers are stored in TiddlerFiles directly beneath the `tiddlers` folder, unless [[configured otherwise|Customising Tiddler File Naming]].

Sub-folders within the `tiddlers` folder can also be given a `tiddlywiki.files` JSON file that overrides the default processing for that folder. The file format is illustrated with this example:

```
{
"tiddlers": [
{
"file": "d3.min.js",
"fields": {
"type": "application/javascript",
"title": "$:/plugins/tiddlywiki/d3/d3.js",
"module-type": "library"
},
"prefix": "var d3;if($tw.browser){\n",
"suffix": "}\nexports.d3 = d3;\n"
},
{
"file": "cloud/d3.layout.cloud.js",
"fields": {
"type": "application/javascript",
"title": "$:/plugins/tiddlywiki/d3/d3.layout.cloud.js",
"module-type": "library"
}
},
{
"file": "local/mytiddler.tid",
"isTiddlerFile": true,
"fields": {
"title": "A different title"
}
}
],
"directories": [
"../../mytiddlers/store"
"tiddlywiki/slider",
"tiddlytools/chooser"
]
}
```

The JSON data consists of an object with a `tiddlers` property that contains an array of information about each tiddler to be loaded into the wiki. That information consists of:
Plugins names refer to plugin folders listed in TiddlyWiki5's root `plugins` folder. Plugins can also be included manually by copying them into the `plugins` subfolder of the wiki.

* The relative or absolute path to the file to include as either:
** `file`: the file containing the tiddler
** `isTiddlerFile`: if `true`, the file will be deserialised to extract the tiddlers. Otherwise, the raw content of the file is assigned to the `text` field without any parsing
* `fields`: an object containing fields that override any provided in the tiddler file
* `prefix` & `suffix`: (optional) specify strings to be prefixed and suffixed to the tiddler file text content
!! Processing of `tiddlers` folder

All the TiddlerFiles in the `tiddlers` folder are read into the wiki at startup. Sub-folders are scanned recursively for TiddlerFiles. Newly created tiddlers are stored in TiddlerFiles directly beneath the `tiddlers` folder, unless [[configured otherwise|Customising Tiddler File Naming]].

The ''directories'' property allows a list of directories to be specified. All tiddlers will be recursively loaded from each directory listed.
The default processing for sub-folders within the `tiddlers` folder can be overridden by providing a JSON file called `tiddlywiki.files` -- see [[tiddlywiki.files Files]].
Loading

0 comments on commit 1b41b44

Please sign in to comment.