Migrate to AppEngine gen2 on java17 with Micronaut and Handlebars (#7)

.github/workflows/CI-build.yml

@@ -50,6 +50,7 @@ jobs:
           --continue
           --no-build-cache
           build
+          :AppEngine:appengineStage
 
       - name: "Upload 'Unit Test Results' artifact."
         if: success() || failure()

AppEngine/README.md

@@ -0,0 +1,22 @@
+## Running a datastore locally
+
+ * Docs: https://cloud.google.com/datastore/docs/tools/datastore-emulator#windows
+ * Old docs: https://cloud.google.com/appengine/docs/legacy/standard/java/tools/using-local-server#datastore
+
+1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/).
+2. Install the emulator:
+   ```shell
+   gcloud components install cloud-datastore-emulator
+   gcloud components install beta
+   ```
+3. Run the datastore emulator in the background:
+   ```
+   gcloud --project twisterrob-travel beta emulators datastore start --no-store-on-disk
+   ```
+4. In a separate window run:
+   ```
+   gcloud beta emulators datastore env-init
+   ```
+5. Set the output environment variables before running `gradlew :AppEngine:run`.
+
+Note: step 4 and 5 are automated in `build.gradle`. 

AppEngine/build.gradle

@@ -1,87 +1,109 @@
+import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform
+
 apply plugin: 'java'
-apply plugin: 'war'
-apply plugin: 'com.google.cloud.tools.appengine-appenginewebxml'
-//apply plugin: 'com.google.cloud.tools.endpoints-framework-server'
+apply plugin: 'io.micronaut.minimal.application'
+apply plugin: 'com.google.cloud.tools.appengine-appyaml'
 apply plugin: 'idea'
 
 java {
-	sourceCompatibility = JavaVersion.VERSION_1_8
-	targetCompatibility = JavaVersion.VERSION_1_8
-}
-
-configurations {
-	dev
-}
-idea {
-	module {
-		scopes.PROVIDED.plus += [ configurations.dev ]
-	}
+	sourceCompatibility = JavaVersion.VERSION_17
+	targetCompatibility = JavaVersion.VERSION_17
 }
 
 dependencies {
-	// com.google.appengine.api.datastore
-	implementation("com.google.appengine:appengine-api-1.0-sdk:${VERSION_APPENGINE}")
-	// com.google.api.server.spi.EndpointsServlet
-	implementation("com.google.endpoints:endpoints-framework:${VERSION_GCLOUD_ENDPOINTS}")
+	implementation("com.google.appengine:appengine-api-1.0-sdk:2.0.12")
+	implementation(platform("com.google.cloud:libraries-bom:${VERSION_GCLOUD}"))
+	implementation("com.google.cloud:google-cloud-datastore")
 
-	providedCompile 'javax.servlet:javax.servlet-api:3.1.0'
-	dev("org.apache.taglibs:taglibs-standard-impl:1.2.5")
-	dev("javax.servlet.jsp.jstl:jstl-api:1.2")
+	runtimeOnly("org.yaml:snakeyaml")
+	runtimeOnly("io.micronaut.serde:micronaut-serde-jackson")
+	implementation("io.micronaut.views:micronaut-views-handlebars")
 
 	implementation project(':Shared')
 
-	// See java.util.logging.properties and log4j2.xml
 	implementation("org.slf4j:slf4j-api:${VERSION_SLF4J}")
-	// route apps SLF4J logging to JUL
-	implementation("org.slf4j:slf4j-jdk14:${VERSION_SLF4J}")
-	dev("org.slf4j:jul-to-slf4j:${VERSION_SLF4J}")
-	dev("org.apache.logging.log4j:log4j-api:${VERSION_LOG4J}")
-	dev("org.apache.logging.log4j:log4j-core:${VERSION_LOG4J}")
-	dev("org.apache.logging.log4j:log4j-slf4j-impl:${VERSION_LOG4J}")
-
+	runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:${VERSION_LOG4J}")
 	// for org.apache.tools.ant.filters.StringInputStream
 	implementation("ant:ant:1.6.5")
 
 	apply from: "${rootDir}/gradle/testCompile.gradle", to: project
-	testImplementation("com.google.appengine:appengine-testing:${VERSION_APPENGINE}")
-	testImplementation("com.google.appengine:appengine-api-stubs:${VERSION_APPENGINE}")
-	testImplementation("com.google.appengine:appengine-api-labs:${VERSION_APPENGINE}")
+	testImplementation("io.micronaut:micronaut-http-client")
+	testImplementation("com.github.jtidy:jtidy:${VERSION_JTIDY}")
 }
 
 sourceSets {
 	main {
 		java.srcDir 'src/main/diff'
-		compileClasspath += configurations.dev
 	}
 	test {
 		java.srcDir 'src/test/diff'
-		compileClasspath += configurations.dev
 	}
 }
 
-appengine { com.google.cloud.tools.gradle.appengine.standard.AppEngineStandardExtension ext ->
+//noinspection UnnecessaryQualifiedReference
+appengine { com.google.cloud.tools.gradle.appengine.appyaml.AppEngineAppYamlExtension ext ->
+	stage {
+		artifact = file("src/main/appengine/placeholder.txt")
+		// AppEngine plugin can't handle laziness, wire manually.
+		tasks.appengineStage.dependsOn(tasks.installDist)
+		extraFilesDirectories = tasks.installDist
+	}
 	deploy {
+		// Live: https://twisterrob-london.appspot.com/
 		projectId = "twisterrob-london"
 		version = "GCLOUD_CONFIG"
-	}
-	run {
-		port = 8888
-		automaticRestart = true
-
-		def logDependencyFilter = { Dependency d -> d.group?.contains('slf4j') || d.group?.contains('log4j') };
-		def deps = configurations.dev.dependencies.findAll logDependencyFilter
-		def flags = configurations.dev.files(deps as Dependency[]) \
-				.collect { "-Xbootclasspath/p:${it.absolutePath}".toString() }
-		jvmFlags = (jvmFlags?: [ ]) + flags
+		// Test deployment:
+		//version = "test"
+		//stopPreviousVersion = false
+		//promote = false
 	}
 	tools {
-		cloudSdkHome = System.getenv("GCLOUD_HOME")?: file("build/downloaded/gcloud-sdk")
+		System.getenv("GCLOUD_HOME")?.tap { cloudSdkHome = it }
 		cloudSdkVersion = VERSION_GCLOUD_SDK as String
 		// https://cloud.google.com/sdk/gcloud/reference#--verbosity
 		verbosity = "info"
 	}
 }
 
-//endpointsServer {
-//	hostname = "twisterrob-london.appspot.com"
-//}
+application {
+	mainClass.set("net.twisterrob.blt.gapp.Application")
+}
+
+micronaut {
+	version = VERSION_MICRONAUT as String
+	runtime("jetty")
+	testRuntime("junit4")
+	processing {
+		incremental(true)
+		annotations("net.twisterrob.blt.gapp.*")
+	}
+}
+
+tasks.named("run").configure { JavaExec task ->
+	if (DefaultNativePlatform.getCurrentOperatingSystem().isWindows()) {
+		task.systemProperty("log4j.skipJansi", false)
+	}
+	if (gradle.startParameter.continuous) {
+		task.systemProperties(
+				// TODO use https://docs.micronaut.io/latest/guide/index.html#environments to create overrides for debug.
+				"micronaut.io.watch.enabled": true,
+				"micronaut.io.watch.restart": true,
+				"micronaut.io.watch.paths": "src/main",
+		)
+	}
+	doFirst {
+		def stream = new ByteArrayOutputStream()
+		exec {
+			commandLine("gcloud.cmd", "beta", "emulators", "datastore", "env-init")
+			standardOutput = stream
+		}
+		Map<String, String> env = stream.toString().split("\r?\n").collectEntries { line ->
+			if (!line.startsWith("set ")) {
+				throw new IllegalStateException("Unexpected line: $line in \n${stream.toString()}")
+			}
+			def (key, value) = line.substring(4).split("=", 2)
+			[ key, value ]
+		}
+		task.environment(env)
+	}
+}

AppEngine/src/main/appengine/app.yaml

@@ -0,0 +1,15 @@
+# https://cloud.google.com/appengine/docs/standard/reference/app-yaml?tab=java      
+runtime: java17
+entrypoint: bin/AppEngine
+
+instance_class: F1
+
+automatic_scaling:
+  min_instances: 0
+  max_instances: 1
+
+handlers:
+  - url: /.*
+    secure: always
+    redirect_http_response_code: 301
+    script: auto

AppEngine/src/main/appengine/placeholder.txt

@@ -0,0 +1,2 @@
+This file exists to replace the mandatory "artifact" in AppEngine YAML plugin.
+The real files come from the `org.gradle.application` plugin's `installDist` task (build/install/AppEngine).

AppEngine/src/main/diff/name/fraser/neil/plaintext/diff_match_patch.java

@@ -1794,6 +1794,7 @@ public LinkedList<Patch> patch_make(LinkedList<Diff> diffs) {
    * @return LinkedList of Patch objects.
    * @deprecated Prefer patch_make(String text1, LinkedList<Diff> diffs).
    */
+  @Deprecated
   public LinkedList<Patch> patch_make(String text1, String text2,
       LinkedList<Diff> diffs) {
     return patch_make(text1, diffs);

AppEngine/src/main/java/net/twisterrob/blt/gapp/Application.java

@@ -0,0 +1,17 @@
+package net.twisterrob.blt.gapp;
+
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.runtime.Micronaut;
+
+public class Application {
+	public static void main(String... args) {
+		Micronaut micronaut = Micronaut
+				.build(args)
+				.classes(Application.class)
+				.banner(false)
+				;
+		try (ApplicationContext context = micronaut.start()) {
+			// Nothing yet, wrap in try-with-resources to ensure teardown.
+		}
+	}
+}

AppEngine/src/main/java/net/twisterrob/blt/gapp/FeedCronServlet.java

@@ -4,27 +4,32 @@
 import java.net.*;
 import java.util.*;
 
-import javax.servlet.http.*;
+import io.micronaut.http.annotation.Controller;
+import io.micronaut.http.annotation.Get;
+import jakarta.servlet.http.*;
 
 import org.slf4j.*;
 
-import com.google.appengine.api.datastore.*;
-import com.google.appengine.api.datastore.Query.SortDirection;
+import com.google.cloud.Timestamp;
+import com.google.cloud.datastore.*;
+import com.google.cloud.datastore.StructuredQuery.OrderBy;
 
 import net.twisterrob.blt.io.feeds.Feed;
 import net.twisterrob.java.io.IOTools;
 import net.twisterrob.java.utils.ObjectTools;
 
 import static net.twisterrob.blt.gapp.FeedConsts.*;
 
+@Controller
 @SuppressWarnings("serial")
 public class FeedCronServlet extends HttpServlet {
 	private static final Logger LOG = LoggerFactory.getLogger(FeedCronServlet.class);
 
 	private static final String QUERY_FEED = "feed";
 
-	private final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
+	private final Datastore datastore = DatastoreOptions.getDefaultInstance().getService();
 
+	@Get("/FeedCron")
 	@Override public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 		String feedString = String.valueOf(req.getParameter(QUERY_FEED));
 		Feed feed;
@@ -39,7 +44,7 @@ public class FeedCronServlet extends HttpServlet {
 		}
 		Marker marker = MarkerFactory.getMarker(feed.name());
 
-		Entity newEntry = downloadNewEntry(feed);
+		FullEntity<IncompleteKey> newEntry = downloadNewEntry(datastore, feed);
 		Entity oldEntry = readLatest(datastore, feed);
 		if (oldEntry != null) {
 			if (sameProp(DS_PROP_CONTENT, oldEntry, newEntry)) {
@@ -60,29 +65,49 @@ public class FeedCronServlet extends HttpServlet {
 		}
 	}
 
-	private static Entity readLatest(DatastoreService datastore, Feed feed) {
-		// we're only concerned about the latest one, if any
-		Query q = new Query(feed.name()).addSort(DS_PROP_RETRIEVED_DATE, SortDirection.DESCENDING);
-		Iterator<Entity> result = datastore.prepare(q).asIterator();
+	private static Entity readLatest(Datastore datastore, Feed feed) {
+		Query<Entity> q = Query
+				.newEntityQueryBuilder()
+				.setKind(feed.name())
+				.addOrderBy(OrderBy.desc(DS_PROP_RETRIEVED_DATE))
+				.build()
+				;
+		// We're only concerned about the latest one, if any.
+		QueryResults<Entity> result = datastore.run(q);
 		return result.hasNext()? result.next() : null;
 	}
 
-	private static boolean sameProp(String propName, Entity oldEntry, Entity newEntry) {
-		return oldEntry.hasProperty(propName) && newEntry.hasProperty(propName)
-				&& ObjectTools.equals(oldEntry.getProperty(propName), newEntry.getProperty(propName));
+	private static boolean sameProp(String propName, BaseEntity<?> oldEntry, BaseEntity<?> newEntry) {
+		return hasProperty(oldEntry, propName) && hasProperty(newEntry, propName)
+				&& ObjectTools.equals(oldEntry.getValue(propName), newEntry.getValue(propName));
 	}
 
-	public static Entity downloadNewEntry(Feed feed) {
-		Entity newEntry = new Entity(feed.name());
+	static boolean hasProperty(BaseEntity<?> entry, String propName) {
+		return entry.getProperties().containsKey(propName);
+	}
+
+	public static FullEntity<IncompleteKey> downloadNewEntry(Datastore datastore, Feed feed) {
+		KeyFactory keyFactory = datastore.newKeyFactory().setKind(feed.name());
+		FullEntity.Builder<IncompleteKey> newEntry = Entity.newBuilder(keyFactory.newKey());
 		try {
 			String feedResult = downloadFeed(feed);
-			newEntry.setProperty(DS_PROP_CONTENT, new Text(feedResult));
+			newEntry.set(DS_PROP_CONTENT, unindexedString(feedResult));
 		} catch (Exception ex) {
 			LOG.error("Cannot load '{}'!", feed, ex);
-			newEntry.setProperty(DS_PROP_ERROR, new Text(ObjectTools.getFullStackTrace(ex)));
+			newEntry.set(DS_PROP_ERROR, unindexedString(ObjectTools.getFullStackTrace(ex)));
 		}
-		newEntry.setProperty(DS_PROP_RETRIEVED_DATE, new Date());
-		return newEntry;
+		newEntry.set(DS_PROP_RETRIEVED_DATE, Timestamp.now());
+		return newEntry.build();
+	}
+
+	/**
+	 * Strings have a limitation of 1500 bytes when indexed. This removes that limitation.
+	 * @see https://cloud.google.com/datastore/docs/concepts/entities#text_string
+	 */
+	private static Value<?> unindexedString(String value) {
+		return value == null
+				? NullValue.of()
+				: StringValue.newBuilder(value).setExcludeFromIndexes(true).build();
 	}
 
 	public static String downloadFeed(Feed feed) throws IOException {

AppEngine/src/main/java/net/twisterrob/blt/gapp/IndexController.java

@@ -0,0 +1,35 @@
+package net.twisterrob.blt.gapp;
+
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.MutableHttpResponse;
+import io.micronaut.http.annotation.Controller;
+import io.micronaut.http.annotation.Get;
+import io.micronaut.http.server.types.files.StreamedFile;
+import io.micronaut.views.View;
+
+import net.twisterrob.blt.gapp.viewmodel.Versions;
+
+@Controller
+public class IndexController {
+
+	@Get("/")
+	@View("index")
+	public MutableHttpResponse<?> index() {
+		return HttpResponse.ok(new IndexModel(new Versions()));
+	}
+
+	@Get("/favicon.ico")
+	public StreamedFile favicon() {
+		return new StreamedFile(
+				IndexController.class.getClassLoader().getResourceAsStream("public/favicon.ico"),
+				MediaType.IMAGE_PNG_TYPE
+		);
+	}
+
+	private record IndexModel(
+			Versions versions
+	) {
+
+	}
+}