Skip to content

Commit

Permalink
Work on #242. This closes #186, closes #175, closes #185, closes 161,
Browse files Browse the repository at this point in the history 
closes #166, closes #154, closes #135, closes #138 and closes #104.
  • Loading branch information
@jonbullock
jonbullock committed Nov 18, 2015
2 parents 84b8f3d + 3e9d8a3 commit 698cd7a
Show file tree
Hide file tree
Showing 114 changed files with 3,601 additions and 1,581 deletions.
701 changes: 391 additions & 310 deletions pom.xml
View file

Large diffs are not rendered by default.

19 changes: 19 additions & 0 deletions src/main/assembly/assembly-example-project-jade.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<assembly>
<id>base</id>
<includeBaseDirectory>false</includeBaseDirectory>
<!-- Generates a zip package containing the needed files -->
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}/example-project-jade/</directory>
<outputDirectory>/</outputDirectory>
<excludes>
<exclude>.git</exclude>
<exclude>README.md</exclude>
<exclude>LICENSE</exclude>
</excludes>
</fileSet>
</fileSets>
</assembly>
19 changes: 19 additions & 0 deletions src/main/assembly/assembly-example-project-pebble.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<assembly>
<id>base</id>
<includeBaseDirectory>false</includeBaseDirectory>
<!-- Generates a zip package containing the needed files -->
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}/example-project-pebble/</directory>
<outputDirectory>/</outputDirectory>
<excludes>
<exclude>.git</exclude>
<exclude>README.md</exclude>
<exclude>LICENSE</exclude>
</excludes>
</fileSet>
</fileSets>
</assembly>
19 changes: 16 additions & 3 deletions src/main/java/org/jbake/app/ConfigUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.jbake.app;

import java.io.File;

import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;

/**
* Provides Configuration related functions.
*
Expand All @@ -21,7 +21,6 @@ public class ConfigUtil {
*
*/
public static interface Keys {

/**
* Output filename for archive file, is only used when {@link #RENDER_ARCHIVE} is true
*/
Expand Down Expand Up @@ -52,6 +51,11 @@ public static interface Keys {
*/
static final String ASSET_FOLDER = "asset.folder";

/**
* URI prefix for content that should be given extensionless output URI's
*/
static final String URI_NO_EXTENSION = "uri.noExtension";

/**
* Flag indicating if hidden asset resources should be ignored
*/
Expand Down Expand Up @@ -182,6 +186,15 @@ public static interface Keys {
*/
static final String VERSION = "version";

/**
* Flag indicating if there should be pagination when rendering index
*/
static final String PAGINATE_INDEX = "index.paginate";

/**
* How many posts per page on index
*/
static final String POSTS_PER_PAGE = "index.posts_per_page";
}

private final static Logger LOGGER = LoggerFactory.getLogger(ConfigUtil.class);
Expand Down
58 changes: 56 additions & 2 deletions src/main/java/org/jbake/app/ContentStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@
import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;

import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.jbake.model.DocumentTypes;

Expand All @@ -42,7 +45,18 @@
* @author jdlee
*/
public class ContentStore {
public static final String PUBLISHED_DATE = "published_date";
public static final String TAG_POSTS = "tag_posts";
public static final String ALLTAGS = "alltags";
public static final String ALL_CONTENT = "all_content";
public static final String PUBLISHED_CONTENT = "published_content";
public static final String PUBLISHED_PAGES = "published_pages";
public static final String PUBLISHED_POSTS = "published_posts";
public static final String DATABASE = "db";

private ODatabaseDocumentTx db;
private long start = -1;
private long limit = -1;

public ContentStore(final String type, String name) {
db = new ODatabaseDocumentTx(type + ":" + name);
Expand All @@ -57,6 +71,27 @@ public ContentStore(final String type, String name) {
}
}

public long getStart() {
return start;
}

public void setStart(int start) {
this.start = start;
}

public long getLimit() {
return limit;
}

public void setLimit(int limit) {
this.limit = limit;
}

public void resetPagination() {
this.start = -1;
this.limit = -1;
}

public final void updateSchema() {
OSchema schema = db.getMetadata().getSchema();
for (String docType : DocumentTypes.getDocumentTypes()) {
Expand All @@ -74,6 +109,7 @@ public final void updateSchema() {

public void close() {
db.close();
DBUtil.closeDataStore();
}

public void drop() {
Expand Down Expand Up @@ -102,11 +138,19 @@ public List<ODocument> getPublishedPages() {
}

public List<ODocument> getPublishedContent(String docType) {
return query("select * from " + docType + " where status='published' order by date desc");
String query = "select * from " + docType + " where status='published'";
if ((start >= 0) && (limit > -1)) {
query += " SKIP " + start + " LIMIT " + limit;
}
return query(query + " order by date desc");
}

public List<ODocument> getAllContent(String docType) {
return query("select * from " + docType + " order by date desc");
String query = "select * from " + docType;
if ((start >= 0) && (limit > -1)) {
query += " SKIP " + start + " LIMIT " + limit;
}
return query(query + " order by date desc");
}

public List<ODocument> getAllTagsFromPublishedPosts() {
Expand Down Expand Up @@ -153,6 +197,16 @@ private void executeCommand(String query, Object... args) {
db.command(new OCommandSQL(query)).execute(args);
}

public Set<String> getTags() {
List<ODocument> docs = this.getAllTagsFromPublishedPosts();
Set<String> result = new HashSet<String>();
for (ODocument document : docs) {
String[] tags = DBUtil.toStringArray(document.field("tags"));
Collections.addAll(result, tags);
}
return result;
}

private static void createDocType(final OSchema schema, final String doctype) {
OClass page = schema.createClass(doctype);
page.createProperty("sha1", OType.STRING).setNotNull(true);
Expand Down
94 changes: 69 additions & 25 deletions src/main/java/org/jbake/app/Crawler.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
package org.jbake.app;

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;

import org.apache.commons.configuration.CompositeConfiguration;
import org.jbake.app.ConfigUtil.Keys;
import org.jbake.app.Crawler.Attributes.Status;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.io.File.separator;

import java.io.File;
import java.util.Arrays;
Expand All @@ -20,14 +19,47 @@
import java.util.Map;
import java.util.Set;

import static java.io.File.separator;
import org.apache.commons.io.FilenameUtils;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;

/**
* Crawls a file system looking for content.
*
* @author Jonathan Bullock <jonbullock@gmail.com>
*/
public class Crawler {
public static interface Attributes {
/**
* Possible values of the {@link Attributes#STATUS} property
* @author ndx
*
*/
public static interface Status {

static final String PUBLISHED_DATE = "published-date";
static final String PUBLISHED = "published";
static final String DRAFT = "draft";
}
static final String CACHED = "cached";
static final String DATE = "date";
static final String STATUS = "status";
static final String TYPE = "type";
static final String URI = "uri";
static final String FILE = "file";
static final String TAGS = "tags";
static final String TAG = "tag";
static final String RENDERED = "rendered";
static final String SHA1 = "sha1";
static final String ROOTPATH = "rootpath";

}
private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class);

private CompositeConfiguration config;
Expand Down Expand Up @@ -105,7 +137,13 @@ private String buildHash(final File sourceFile) {

private String buildURI(final File sourceFile) {
String uri = FileUtil.asPath(sourceFile.getPath()).replace(FileUtil.asPath( contentPath), "");
// strip off leading / to enable generating non-root based sites
String noExtensionUrlFolder = config.getString(Keys.URI_NO_EXTENSION);
if (!noExtensionUrlFolder.equals("false") && uri.startsWith(noExtensionUrlFolder)) {
uri = "/" + FilenameUtils.getPath(uri) + FilenameUtils.getBaseName(uri) + "/index" + config.getString(Keys.OUTPUT_EXTENSION);
} else {
uri = uri.substring(0, uri.lastIndexOf(".")) + config.getString(Keys.OUTPUT_EXTENSION);
}
// strip off leading / to enable generating non-root based sites
if (uri.startsWith("/")) {
uri = uri.substring(1, uri.length());
}
Expand All @@ -115,29 +153,35 @@ private String buildURI(final File sourceFile) {
private void crawlSourceFile(final File sourceFile, final String sha1, final String uri) {
Map<String, Object> fileContents = parser.processFile(sourceFile);
if (fileContents != null) {
fileContents.put("rootpath", getPathToRoot(sourceFile));
fileContents.put("sha1", sha1);
fileContents.put("rendered", false);
if (fileContents.get("tags") != null) {
fileContents.put(Attributes.ROOTPATH, getPathToRoot(sourceFile));
fileContents.put(Attributes.SHA1, sha1);
fileContents.put(Attributes.RENDERED, false);
if (fileContents.get(Attributes.TAGS) != null) {
// store them as a String[]
String[] tags = (String[]) fileContents.get("tags");
fileContents.put("tags", tags);
String[] tags = (String[]) fileContents.get(Attributes.TAGS);
fileContents.put(Attributes.TAGS, tags);
}
fileContents.put("file", sourceFile.getPath());
fileContents.put("uri", uri.substring(0, uri.lastIndexOf(".")) + FileUtil.findExtension(config, fileContents.get("type").toString()));

String documentType = (String) fileContents.get("type");
if (fileContents.get("status").equals("published-date")) {
if (fileContents.get("date") != null && (fileContents.get("date") instanceof Date)) {
if (new Date().after((Date) fileContents.get("date"))) {
fileContents.put("status", "published");

fileContents.put(Attributes.FILE, sourceFile.getPath());
fileContents.put(Attributes.URI, uri);

String documentType = (String) fileContents.get(Attributes.TYPE);
if (fileContents.get(Attributes.STATUS).equals(Status.PUBLISHED_DATE)) {
if (fileContents.get(Attributes.DATE) != null && (fileContents.get(Attributes.DATE) instanceof Date)) {
if (new Date().after((Date) fileContents.get(Attributes.DATE))) {
fileContents.put(Attributes.STATUS, Status.PUBLISHED);
}
}
}

if (!config.getString(Keys.URI_NO_EXTENSION).equals("false")) {
fileContents.put("noExtensionUri", uri.replace("/index.html", "/"));
}

ODocument doc = new ODocument(documentType);
doc.fields(fileContents);
boolean cached = fileContents.get("cached") != null ? Boolean.valueOf((String)fileContents.get("cached")):true;
doc.field("cached", cached);
boolean cached = fileContents.get(Attributes.CACHED) != null ? Boolean.valueOf((String)fileContents.get(Attributes.CACHED)):true;
doc.field(Attributes.CACHED, cached);
doc.save();
} else {
LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile);
Expand All @@ -162,12 +206,12 @@ public String getPathToRoot(File sourceFile) {
public int getDocumentCount(String docType) {
return (int) db.countClass(docType);
}

public Set<String> getTags() {
List<ODocument> query = db.getAllTagsFromPublishedPosts(); //query(new OSQLSynchQuery<ODocument>("select tags from post where status='published'"));
Set<String> result = new HashSet<String>();
for (ODocument document : query) {
String[] tags = DBUtil.toStringArray(document.field("tags"));
String[] tags = DBUtil.toStringArray(document.field(Attributes.TAGS));
Collections.addAll(result, tags);
}
return result;
Expand All @@ -177,8 +221,8 @@ private DocumentStatus findDocumentStatus(String docType, String uri, String sha
List<ODocument> match = db.getDocumentStatus(docType, uri);
if (!match.isEmpty()) {
ODocument entries = match.get(0);
String oldHash = entries.field("sha1");
if (!(oldHash.equals(sha1)) || Boolean.FALSE.equals(entries.field("rendered"))) {
String oldHash = entries.field(Attributes.SHA1);
if (!(oldHash.equals(sha1)) || Boolean.FALSE.equals(entries.field(Attributes.RENDERED))) {
return DocumentStatus.UPDATED;
} else {
return DocumentStatus.IDENTICAL;
Expand Down
15 changes: 11 additions & 4 deletions src/main/java/org/jbake/app/DBUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,25 @@
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import org.jbake.model.DocumentTypes;

import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

public class DBUtil {
public static ContentStore createDB(final String type, String name) {
return new ContentStore(type, name);
}
private static ContentStore contentStore;

public static ContentStore createDataStore(final String type, String name) {
return new ContentStore(type, name);
if (contentStore == null) {
contentStore = new ContentStore(type, name);
}
return contentStore;
}

public static void closeDataStore() {
contentStore = null;
}

public static void updateSchema(final ContentStore db) {
Expand Down
Loading

0 comments on commit 698cd7a

Please sign in to comment.