Make it easier to use DataGrip w/ Synapse's schema

contrib/datagrip/README.md

@@ -0,0 +1,28 @@
+# Schema symlinks
+
+This directory contains symlinks to the latest dump of the postgres full schema. This is useful to have, as it allows IDEs to understand our schema and provide autocomplete, linters, inspections, etc.
+
+I (dmr) use the DataGrip functionality in IntelliJ's paid-for products. It seems to only consider files called `*.sql` when defining a schema from DDL; `*.sql.postgres` wasn't detected for me.
+
+## Example
+![](datagrip-aware-of-schema.png)
+
+## Caveats
+
+- Doesn't include temporary tables created ad-hoc by Synapse.
+- Postgres only. IDEs will likely be confused by SQLite-specific queries.
+- Will not include migrations created after the latest schema dump.
+- Symlinks might confuse checkouts on Windows systems.
+
+## Instructions
+
+### Jetbrains IDEs with DataGrip plugin
+
+- View -> Tool Windows -> Database
+- `+` Icon -> DDL Data Source
+- Pick a name, e.g. `Synapse schema dump`
+- Under sources, click `+`.
+- Add an entry with Path pointing to this directory, and dialect set to PostgreSQL.
+- OK, and OK.
+- IDE should now be aware of the schema.
+- Try control-clicking on a table name in a bit of SQL e.g. in `_get_forgotten_rooms_for_user_txn`.

contrib/datagrip/common.sql

@@ -0,0 +1 @@
+../../synapse/storage/schema/common/full_schemas/72/full.sql.postgres

contrib/datagrip/main.sql

@@ -0,0 +1 @@
+../../synapse/storage/schema/main/full_schemas/72/full.sql.postgres

contrib/datagrip/schema_version.sql

@@ -0,0 +1 @@
+../../synapse/storage/schema/common/schema_version.sql

contrib/datagrip/state.sql

@@ -0,0 +1 @@
+../../synapse/storage/schema/state/full_schemas/72/full.sql.postgres

pyproject.toml

@@ -124,7 +124,10 @@ include = [
     { path = "rust/src/**", format = "sdist" },
 ]
 exclude = [
-    { path = "synapse/*.so", format = "sdist"}
+    { path = "synapse/*.so", format = "sdist" },
+    # This contains dev-only symlinks to the latest schema dumps. We don't want people
+    # to use it when installing.
+    { path = "synapse/storage/schema/latest_schema" },
 ]
 
 build = "build_rust.py"

scripts-dev/make_full_schema.sh

@@ -19,14 +19,20 @@ usage() {
   echo "-c"
   echo "  CI mode. Prints every command that the script runs."
   echo "-o <path>"
-  echo "  Directory to output full schema files to."
+  echo "  Directory to output full schema files to. You probably want to use"
+  echo "  '-o synapse/storage/schema'"
   echo "-n <schema number>"
   echo "  Schema number for the new snapshot. Used to set the location of files within "
   echo "  the output directory, mimicking that of synapse/storage/schemas."
   echo "  Defaults to 9999."
   echo "-h"
   echo "  Display this help text."
   echo ""
+  echo ""
+  echo "You probably want to invoke this with something like"
+  echo "  docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:11-alpine"
+  echo "  echo postgres | scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema"
+  echo ""
   echo "  NB: make sure to run this against the *oldest* supported version of postgres,"
   echo "  or else pg_dump might output non-backwards-compatible syntax."
 }
@@ -189,7 +195,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 echo "Running db background jobs..."
-synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
+poetry run python synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
 
 # Create the PostgreSQL database.
 echo "Creating postgres databases..."
@@ -198,7 +204,7 @@ createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_MAIN_DB_NAM
 createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_STATE_DB_NAME"
 
 echo "Running db background jobs..."
-synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
+poetry run python synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
 
 
 echo "Dropping unwanted db tables..."
@@ -293,4 +299,12 @@ pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owne
 pg_dump --format=plain --schema-only         --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME"  | cleanup_pg_schema  > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
 pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME"  | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
 
+if [ "$OUTPUT_DIR" == *synapse/storage/schema ]; then
+  echo "Updating contrib/datagrip symlinks..."
+  ln -sf "synapse/storage/schema/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" "contrib/datagrip/common.sql"
+  ln -sf "synapse/storage/schema/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"   "contrib/datagrip/main.sql"
+  ln -sf "synapse/storage/schema/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"  "contrib/datagrip/state.sql"
+else
+  echo "Not updating contrib/datagrip symlinks (unknown output directory)"
+fi
 echo "Done! Files dumped to: $OUTPUT_DIR"