From cc77ab92ff0c5db5cd182835415d4ff1ee715ec6 Mon Sep 17 00:00:00 2001 From: vinceh121 Date: Fri, 16 Feb 2024 22:43:27 +0100 Subject: [PATCH] fix: some progress towards something working --- gmcserver-server/migrations/202402162056.sql | 110 ++++++++++++++++++ gmcserver-server/pom.xml | 22 +--- .../vinceh121/gmcserver/DatabaseManager.java | 18 ++- .../gmcserver/auth/InternalAuthenticator.java | 7 +- .../gmcserver/entities/AbstractEntity.java | 14 +++ .../vinceh121/gmcserver/entities/Device.java | 4 + .../gmcserver/entities/DeviceCalendar.java | 4 + .../gmcserver/entities/DeviceStats.java | 4 + .../vinceh121/gmcserver/entities/Record.java | 4 + .../me/vinceh121/gmcserver/entities/User.java | 3 + .../managers/DeviceCalendarManager.java | 2 +- .../gmcserver/managers/DeviceManager.java | 6 +- .../gmcserver/managers/ImportManager.java | 12 +- .../gmcserver/managers/LoggingManager.java | 2 +- .../gmcserver/managers/UserManager.java | 22 ++-- .../src/main/resources/log4j2.xml | 2 +- .../src/test/java/gmcserver/TestMapping.java | 16 +++ 17 files changed, 206 insertions(+), 46 deletions(-) create mode 100644 gmcserver-server/migrations/202402162056.sql create mode 100644 gmcserver-server/src/test/java/gmcserver/TestMapping.java diff --git a/gmcserver-server/migrations/202402162056.sql b/gmcserver-server/migrations/202402162056.sql new file mode 100644 index 0000000..ca73959 --- /dev/null +++ b/gmcserver-server/migrations/202402162056.sql @@ -0,0 +1,110 @@ +-- DROP SCHEMA public; + +CREATE SCHEMA public AUTHORIZATION postgres; + +COMMENT ON SCHEMA public IS 'standard public schema'; +-- public.users definition + +-- Drop table + +-- DROP TABLE public.users; + +CREATE TABLE public.users ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + username varchar NOT NULL, + "password" varchar NULL, + email varchar NOT NULL, + "deviceLimit" int4 NOT NULL, + "gmcId" int8 NOT NULL, + "admin" bool NOT NULL, + mfa bool NOT NULL, + "alertEmails" bool NOT NULL, + "mfaKey" jsonb NULL, + CONSTRAINT users_pk PRIMARY KEY (id) +); +CREATE INDEX users_email_idx ON public.users USING btree (email); +CREATE UNIQUE INDEX users_gmcid_idx ON public.users USING btree ("gmcId"); +CREATE INDEX users_username_idx ON public.users USING btree (username); + + +-- public.calendar definition + +-- Drop table + +-- DROP TABLE public.calendar; + +CREATE TABLE public.calendar ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + "deviceId" uuid NOT NULL, + "createdAt" timestamp NOT NULL, + recs jsonb NOT NULL, + "inProgress" bool NOT NULL +); +COMMENT ON TABLE public.calendar IS 'Single day averages of devices'; + + +-- public.devices definition + +-- Drop table + +-- DROP TABLE public.devices; + +CREATE TABLE public.devices ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + model varchar NULL, + "name" varchar NULL, + importedfrom varchar NULL, + "location" point NULL, + "owner" uuid NOT NULL, + lastrecordid uuid NULL, + gmcid int8 NULL, + disabled bool NOT NULL, + lastemailalert timestamp NOT NULL, + stddevalertlimit float8 NULL, + proxiessettings jsonb NULL, + CONSTRAINT devices_pk PRIMARY KEY (id) +); +CREATE UNIQUE INDEX devices_gmcid_idx ON public.devices USING btree (gmcid); + + +-- public.records definition + +-- Drop table + +-- DROP TABLE public.records; + +CREATE TABLE public.records ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + "deviceId" uuid NOT NULL, + "date" timestamp NOT NULL, + ip inet NULL, + "type" varchar NULL, + "location" point NULL, + cpm float8 NULL, + acpm float8 NULL, + usv float8 NULL, + co2 float8 NULL, + hcho float8 NULL, + tmp float8 NULL, + ap float8 NULL, + hmdt float8 NULL, + accy float8 NULL, + CONSTRAINT records_pk PRIMARY KEY (id) +); +CREATE INDEX records_deviceid_date_idx ON public.records USING btree ("deviceId", date); + + +-- public.calendar foreign keys + +ALTER TABLE public.calendar ADD CONSTRAINT calendar_devices_fk FOREIGN KEY ("deviceId") REFERENCES public.devices(id) ON DELETE CASCADE; + + +-- public.devices foreign keys + +ALTER TABLE public.devices ADD CONSTRAINT devices_lastrecordid_fk FOREIGN KEY (lastrecordid) REFERENCES public.records(id) ON DELETE CASCADE; +ALTER TABLE public.devices ADD CONSTRAINT devices_owner_fk FOREIGN KEY ("owner") REFERENCES public.users(id) ON DELETE CASCADE; + + +-- public.records foreign keys + +ALTER TABLE public.records ADD CONSTRAINT records_fk FOREIGN KEY ("deviceId") REFERENCES public.devices(id) ON DELETE CASCADE; diff --git a/gmcserver-server/pom.xml b/gmcserver-server/pom.xml index 133b885..73f525d 100644 --- a/gmcserver-server/pom.xml +++ b/gmcserver-server/pom.xml @@ -40,8 +40,9 @@ ./mail.json ./gmcserver-email/out/ - 4.3.8 + 4.5.3 2.15.0 + 2.22.1 @@ -110,53 +111,42 @@ argon2-jvm 2.7 - org.apache.logging.log4j log4j-core - 2.17.1 - + ${log4j.version} + org.apache.logging.log4j log4j-slf4j-impl - 2.17.1 + ${log4j.version} - commons-codec commons-codec 1.14 - org.apache.commons commons-collections4 4.4 - com.eatthepath java-otp 0.2.0 - org.jsoup jsoup 1.15.3 - org.ldaptive ldaptive - 2.1.1 + 2.3.0 - org.junit.jupiter junit-jupiter-engine diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/DatabaseManager.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/DatabaseManager.java index 4173f1f..ad68fad 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/DatabaseManager.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/DatabaseManager.java @@ -19,8 +19,10 @@ import java.util.Map; +import io.vertx.core.tracing.TracingPolicy; +import io.vertx.pgclient.PgBuilder; import io.vertx.pgclient.PgConnectOptions; -import io.vertx.pgclient.PgPool; +import io.vertx.sqlclient.Pool; import io.vertx.sqlclient.PoolOptions; import io.vertx.sqlclient.Row; import io.vertx.sqlclient.RowSet; @@ -29,24 +31,28 @@ import me.vinceh121.gmcserver.managers.AbstractManager; public class DatabaseManager extends AbstractManager { - private final PgPool pool; + private final Pool pool; public DatabaseManager(final GMCServer srv) { super(srv); + final PgConnectOptions optsConfig; - if (srv.getConfig().contains("db.uri")) { + + if (srv.getConfig().containsKey("db.uri")) { optsConfig = PgConnectOptions.fromUri(srv.getConfig().getProperty("db.uri")); } else { optsConfig = new PgConnectOptions(); } final PgConnectOptions optsEnv = PgConnectOptions.fromEnv(); - final PgConnectOptions optsEffective = optsConfig.merge(optsEnv.toJson()); + final PgConnectOptions optsEffective = optsEnv.merge(optsConfig.toJson()); + + optsEffective.setTracingPolicy(TracingPolicy.ALWAYS); final PoolOptions poolOpts = new PoolOptions(); poolOpts.setMaxSize(5); - this.pool = PgPool.pool(srv.getVertx(), optsEffective, poolOpts); + this.pool = PgBuilder.pool().with(poolOpts).connectingTo(optsEffective).build(); this.checkIndexes(); } @@ -55,7 +61,7 @@ public DatabaseManager(final GMCServer srv) { private void checkIndexes() { } - public PgPool getPool() { + public Pool getPool() { return this.pool; } diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/auth/InternalAuthenticator.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/auth/InternalAuthenticator.java index d75a652..930c417 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/auth/InternalAuthenticator.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/auth/InternalAuthenticator.java @@ -20,6 +20,7 @@ import java.util.Collections; import io.vertx.core.Future; +import io.vertx.sqlclient.RowIterator; import me.vinceh121.gmcserver.GMCServer; import me.vinceh121.gmcserver.entities.User; import me.vinceh121.gmcserver.exceptions.AuthenticationException; @@ -45,13 +46,15 @@ public Future login(final String username, final String password) { .mapTo(User.class) .execute(Collections.singletonMap("username", username)) .onSuccess(rowSet -> { - final User user = rowSet.iterator().next(); + final RowIterator iter = rowSet.iterator(); - if (user == null) { + if (!iter.hasNext()) { promise.fail(new EntityNotFoundException("User not found")); return; } + final User user = iter.next(); + if (user.getPassword() == null) { promise.fail(new IllegalStateException("User account disabled")); return; diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/AbstractEntity.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/AbstractEntity.java index 281db2c..89191f9 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/AbstractEntity.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/AbstractEntity.java @@ -17,11 +17,15 @@ */ package me.vinceh121.gmcserver.entities; +import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition; import io.vertx.core.json.JsonObject; +import io.vertx.core.json.jackson.DatabindCodec; public abstract class AbstractEntity { private UUID id = UUID.randomUUID(); @@ -46,4 +50,14 @@ public JsonObject toJson() { public JsonObject toPublicJson() { return this.toJson(); } + + public static String sqlFields(final Class cls) { + final List list = DatabindCodec.mapper() + .writerFor(cls) + .getConfig() + .introspect(DatabindCodec.mapper().constructType(cls)) + .findProperties(); + + return list.stream().map(p -> "#{" + p.getName() + "}").collect(Collectors.joining(",")); + } } diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Device.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Device.java index a8c497a..b15449f 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Device.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Device.java @@ -184,4 +184,8 @@ public JsonObject toMapJson() { obj.put("lastRecord", this.getLastRecord().toPublicJson()); return obj; } + + public static String sqlFields() { + return AbstractEntity.sqlFields(Device.class); + } } diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceCalendar.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceCalendar.java index 193e40b..779c5b2 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceCalendar.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceCalendar.java @@ -59,4 +59,8 @@ public boolean isInProgress() { public void setInProgress(final boolean inProgress) { this.inProgress = inProgress; } + + public static String sqlFields() { + return AbstractEntity.sqlFields(DeviceCalendar.class); + } } diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceStats.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceStats.java index d24ecb4..7e41cbc 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceStats.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/DeviceStats.java @@ -89,4 +89,8 @@ public JsonObject toJson() { obj.remove("id"); return obj; } + + public static String sqlFields() { + return AbstractEntity.sqlFields(DeviceStats.class); + } } diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Record.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Record.java index 07cbed1..24024a2 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Record.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/Record.java @@ -318,6 +318,10 @@ public String toString() { + ", location=" + this.location + "]"; } + public static String sqlFields() { + return AbstractEntity.sqlFields(Record.class); + } + public static class Builder { // XXX this will need a big clean up but at least it splits stuff private final Record record = new Record(); diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/User.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/User.java index 1a06688..5b34e78 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/User.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/entities/User.java @@ -148,4 +148,7 @@ public String toString() { return this.getUsername() + " (" + this.getId().toString() + ")"; } + public static String sqlFields() { + return AbstractEntity.sqlFields(User.class); + } } diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceCalendarManager.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceCalendarManager.java index 5b060d2..b59cde5 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceCalendarManager.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceCalendarManager.java @@ -109,7 +109,7 @@ protected void executeSync(final Promise promise) { cal.setInProgress(true); this.srv.getDatabaseManager() - .update("INSERT INTO calendar VALUES") + .update("INSERT INTO calendar VALUES (" + DeviceCalendar.sqlFields() + ")") .mapFrom(DeviceCalendar.class) .execute(cal) .onSuccess(r -> { diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceManager.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceManager.java index 41cee9f..e62dd10 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceManager.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/DeviceManager.java @@ -612,15 +612,15 @@ protected void executeSync(final Promise promise) { } dev.setLocation(location); - promise.complete(dev); - if (this.insertInDb) { this.srv.getDatabaseManager() - .update("INSERT INTO devices VALUES") + .update("INSERT INTO devices VALUES (" + Device.sqlFields() + ")") .mapFrom(Device.class) .execute(dev) .onSuccess(rs -> promise.complete(dev)) .onFailure(promise::fail); + } else { + promise.complete(dev); } }) .onFailure(promise::fail); diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/ImportManager.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/ImportManager.java index ed47c1e..21cc160 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/ImportManager.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/ImportManager.java @@ -106,7 +106,7 @@ protected void executeSync(Promise promise) { } this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .executeBatch(recs) .onSuccess(res -> { @@ -123,7 +123,7 @@ private void importPageRecurse(final int page) { if (recs.size() != 0) { this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .executeBatch(recs) .onSuccess(res -> { @@ -396,7 +396,7 @@ protected void executeSync(Promise promise) { } this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .executeBatch(recs) .onSuccess(res -> { @@ -415,7 +415,7 @@ private void importPageRecurse(final int page) { if (recs.size() != 0) { this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .executeBatch(recs) .onSuccess(res -> { @@ -693,7 +693,7 @@ protected void executeSync(Promise promise) { } this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .executeBatch(recs) .onSuccess(r -> { @@ -760,7 +760,7 @@ protected void executeSync(final Promise promise) { rec.setDate(date); // TODO batch inserts this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .execute(rec) .onSuccess(r -> { diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/LoggingManager.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/LoggingManager.java index 43a4f99..6ac82bf 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/LoggingManager.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/LoggingManager.java @@ -75,7 +75,7 @@ protected void executeSync(final Promise promise) { if (this.insertInDb) { joinedFutures.add(Future.future(p -> { this.srv.getDatabaseManager() - .update("INSERT INTO records VALUES") + .update("INSERT INTO records VALUES (" + Record.sqlFields() + ")") .mapFrom(Record.class) .execute(this.record) .onSuccess(e -> p.complete()) diff --git a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/UserManager.java b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/UserManager.java index 979a762..ff4d9c7 100644 --- a/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/UserManager.java +++ b/gmcserver-server/src/main/java/me/vinceh121/gmcserver/managers/UserManager.java @@ -20,7 +20,6 @@ import java.security.SecureRandom; import java.security.SignatureException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -32,6 +31,7 @@ import io.vertx.core.CompositeFuture; import io.vertx.core.Future; import io.vertx.core.Promise; +import io.vertx.sqlclient.RowIterator; import io.vertx.sqlclient.Tuple; import me.vinceh121.gmcserver.GMCServer; import me.vinceh121.gmcserver.actions.AbstractAction; @@ -96,14 +96,17 @@ protected void executeSync(final Promise promise) { this.srv.getDatabaseManager() .query("SELECT * FROM users WHERE " + (this.gmcId == null ? "id = #{id}" : "gmcId = #{gmcId}")) .mapTo(User.class) - .execute(Map.of("id", this.id, "gmcId", this.gmcId)) + .execute(this.gmcId == null ? Map.of("id", this.id) : Map.of("gmcId", this.gmcId)) .onSuccess(rowSet -> { - User user = rowSet.iterator().next(); - if (user != null) { - promise.complete(user); - } else { + final RowIterator iter = rowSet.iterator(); + + if (!iter.hasNext()) { promise.fail(new EntityNotFoundException("Failed to get user")); + return; } + + final User user = iter.next(); + promise.complete(user); }) .onFailure(promise::fail); } @@ -256,8 +259,7 @@ protected void executeSync(final Promise promise) { user.setGmcId(this.gmcId); } - @SuppressWarnings("rawtypes") - List checks = new ArrayList<>(2); + List> checks = new ArrayList<>(2); // FIXME rely on UNIQUE constraints if (this.checkUsernameAvailable) { checks.add(Future.future(p -> { @@ -291,10 +293,10 @@ protected void executeSync(final Promise promise) { })); } - CompositeFuture.all(checks).onSuccess(f -> { + Future.all(checks).onSuccess(f -> { if (this.insertInDb) { this.srv.getDatabaseManager() - .update("INSERT INTO users VALUES") + .update("INSERT INTO users VALUES (" + User.sqlFields() + ")") .mapFrom(User.class) .execute(user) .onSuccess(e -> promise.complete(user)) diff --git a/gmcserver-server/src/main/resources/log4j2.xml b/gmcserver-server/src/main/resources/log4j2.xml index 99bf28d..b18c141 100644 --- a/gmcserver-server/src/main/resources/log4j2.xml +++ b/gmcserver-server/src/main/resources/log4j2.xml @@ -1,5 +1,5 @@ - +