Skip to content

Create data migration for moving to new tables #315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 11, 2025
Merged

Create data migration for moving to new tables #315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
421bc08
Create data migration for moving to new tables
erskingardner Aug 11, 2025
0cc6c40
Add relay migration for accounts
erskingardner Aug 11, 2025
d6c85b9
Fix flakey tests
erskingardner Aug 11, 2025
8edbf68
Add uniqueness constraint and update queries
erskingardner Aug 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions db_migrations/0005_new_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@ CREATE TABLE accounts_new (
);

-- New users table
CREATE TABLE users_new (
CREATE TABLE users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
pubkey TEXT NOT NULL UNIQUE,
metadata JSONB,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);

-- New account_followers table
CREATE TABLE account_followers (
-- New account_follows table
CREATE TABLE account_follows (
id INTEGER PRIMARY KEY AUTOINCREMENT,
account_id INTEGER NOT NULL,
user_id INTEGER NOT NULL,
Expand All @@ -30,7 +30,7 @@ CREATE TABLE account_followers (
-- New relays table
CREATE TABLE relays (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL,
url TEXT NOT NULL UNIQUE,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
Expand All @@ -55,20 +55,20 @@ BEGIN
UPDATE accounts_new SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
END;

-- Trigger for users_new table
CREATE TRIGGER update_users_new_updated_at
AFTER UPDATE ON users_new
-- Trigger for users table
CREATE TRIGGER update_users_updated_at
AFTER UPDATE ON users
FOR EACH ROW
BEGIN
UPDATE users_new SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
UPDATE users SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
END;

-- Trigger for account_followers table
CREATE TRIGGER update_account_followers_updated_at
AFTER UPDATE ON account_followers
-- Trigger for account_follows table
CREATE TRIGGER update_account_follows_updated_at
AFTER UPDATE ON account_follows
FOR EACH ROW
BEGIN
UPDATE account_followers SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
UPDATE account_follows SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
END;

-- Trigger for relays table
Expand Down
176 changes: 176 additions & 0 deletions db_migrations/0006_data_migration.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
-- Add unique index to user_relays table to prevent duplicate entries
CREATE UNIQUE INDEX IF NOT EXISTS idx_user_relays_unique
ON user_relays (user_id, relay_id, relay_type);

-- Step 1: Migrate contacts to users table
INSERT INTO users (pubkey, metadata)
SELECT
pubkey,
CASE
WHEN metadata IS NOT NULL AND metadata != '' THEN json(metadata)
ELSE NULL
END as metadata
Comment on lines +9 to +12
Copy link
Contributor

@coderabbitai coderabbitai bot Aug 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Avoid nullifying metadata: only call json() on valid JSON.

json(metadata) returns NULL on invalid JSON, which will drop non-JSON-but-previously-populated metadata. Check json_valid first.

-    CASE
-        WHEN metadata IS NOT NULL AND metadata != '' THEN json(metadata)
-        ELSE NULL
-    END as metadata
+    CASE
+        WHEN json_valid(metadata) THEN json(metadata)
+        ELSE NULL
+    END AS metadata
🤖 Prompt for AI Agents 
In db_migrations/0006_data_migration.sql around lines 5 to 8, the current CASE
calls json(metadata) unconditionally which yields NULL for invalid JSON and
drops existing non-JSON metadata; update the condition to only call
json(metadata) when json_valid(metadata) is true and otherwise preserve the
original metadata (do not return NULL) so invalid/non-JSON strings remain
unchanged.

FROM contacts
WHERE NOT EXISTS (SELECT 1 FROM users WHERE users.pubkey = contacts.pubkey);
Comment on lines +13 to +14
Copy link
Contributor

@coderabbitai coderabbitai bot Aug 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Avoid inserting users with NULL/empty pubkeys.

Rows with NULL/empty pubkey will pass the NOT EXISTS check and attempt to insert, likely violating NOT NULL/PK constraints.

Apply:

-FROM contacts
-WHERE NOT EXISTS (SELECT 1 FROM users WHERE users.pubkey = contacts.pubkey);
+FROM contacts
+WHERE pubkey IS NOT NULL
+  AND pubkey != ''
+  AND NOT EXISTS (SELECT 1 FROM users WHERE users.pubkey = contacts.pubkey);
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
FROM contacts
WHERE NOT EXISTS (SELECT 1 FROM users WHERE users.pubkey = contacts.pubkey);
FROM contacts
WHERE pubkey IS NOT NULL
AND pubkey != ''
AND NOT EXISTS (SELECT 1 FROM users WHERE users.pubkey = contacts.pubkey);
🤖 Prompt for AI Agents 
In db_migrations/0006_data_migration.sql around lines 13 to 14, the INSERT
selects from contacts and will attempt to insert rows where contacts.pubkey is
NULL or empty because the NOT EXISTS check alone doesn't exclude them; update
the WHERE clause to exclude NULL and empty (and optionally whitespace-only)
pubkeys before the NOT EXISTS check (e.g., add conditions like contacts.pubkey
IS NOT NULL AND contacts.pubkey <> '' or use TRIM(contacts.pubkey) <> '') so
only valid non-empty pubkeys are considered for insertion.


-- Step 2: Extract and insert unique relay URLs from contacts
-- Extract from nip65_relays
INSERT OR IGNORE INTO relays (url)
SELECT DISTINCT
json_extract(relay_value.value, '$') as url
FROM contacts,
json_each(contacts.nip65_relays) as relay_value
WHERE json_valid(contacts.nip65_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Extract from inbox_relays
INSERT OR IGNORE INTO relays (url)
SELECT DISTINCT
json_extract(relay_value.value, '$') as url
FROM contacts,
json_each(contacts.inbox_relays) as relay_value
WHERE json_valid(contacts.inbox_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Extract from key_package_relays
INSERT OR IGNORE INTO relays (url)
SELECT DISTINCT
json_extract(relay_value.value, '$') as url
FROM contacts,
json_each(contacts.key_package_relays) as relay_value
WHERE json_valid(contacts.key_package_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Step 3: Create user_relays relationships
-- Insert nip65_relays relationships
INSERT OR IGNORE INTO user_relays (user_id, relay_id, relay_type)
SELECT DISTINCT
u.id as user_id,
r.id as relay_id,
'nip65' as relay_type
FROM contacts c
JOIN users u ON u.pubkey = c.pubkey
JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
json_each(c.nip65_relays) as relay_value
WHERE json_valid(c.nip65_relays)
Comment on lines +56 to +58
Copy link
Contributor

@coderabbitai coderabbitai bot Aug 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Fix JOIN order: ON clause references alias not yet in scope (will error in SQLite).

In each block, the JOIN relays r ... ON ... references relay_value before json_each(...) relay_value is introduced (appears after a comma). In SQLite, the ON clause can only reference tables to its left. Reorder to introduce json_each(...) first (via CROSS JOIN), then join relays.

Apply these diffs:

--- Step 3: nip65_relays
-FROM contacts c
-JOIN users u ON u.pubkey = c.pubkey
-JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
-     json_each(c.nip65_relays) as relay_value
+FROM contacts c
+JOIN users u ON u.pubkey = c.pubkey
+CROSS JOIN json_each(c.nip65_relays) AS relay_value
+JOIN relays r ON r.url = json_extract(relay_value.value, '$')
 WHERE json_valid(c.nip65_relays)
--- Step 3: inbox_relays
-FROM contacts c
-JOIN users u ON u.pubkey = c.pubkey
-JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
-     json_each(c.inbox_relays) as relay_value
+FROM contacts c
+JOIN users u ON u.pubkey = c.pubkey
+CROSS JOIN json_each(c.inbox_relays) AS relay_value
+JOIN relays r ON r.url = json_extract(relay_value.value, '$')
 WHERE json_valid(c.inbox_relays)
--- Step 3: key_package_relays
-FROM contacts c
-JOIN users u ON u.pubkey = c.pubkey
-JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
-     json_each(c.key_package_relays) as relay_value
+FROM contacts c
+JOIN users u ON u.pubkey = c.pubkey
+CROSS JOIN json_each(c.key_package_relays) AS relay_value
+JOIN relays r ON r.url = json_extract(relay_value.value, '$')
 WHERE json_valid(c.key_package_relays)

Also applies to: 66-68, 80-82

🤖 Prompt for AI Agents 
In db_migrations/0006_data_migration.sql around lines 52-54 (also apply same
change at 66-68 and 80-82), the JOIN ordering is incorrect because the ON clause
references the alias relay_value before json_each(... ) AS relay_value is
introduced; reorder the joins so json_each(c.nip65_relays) AS relay_value (as a
CROSS JOIN or comma-separated table placed to the left) is introduced first,
then JOIN relays r ON r.url = json_extract(relay_value.value, '$'); ensure
json_valid(c.nip65_relays) remains in the WHERE clause and update each affected
block accordingly.

AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Insert inbox_relays relationships
INSERT OR IGNORE INTO user_relays (user_id, relay_id, relay_type)
SELECT DISTINCT
u.id as user_id,
r.id as relay_id,
'inbox' as relay_type
FROM contacts c
JOIN users u ON u.pubkey = c.pubkey
JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
json_each(c.inbox_relays) as relay_value
WHERE json_valid(c.inbox_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Insert key_package_relays relationships
INSERT OR IGNORE INTO user_relays (user_id, relay_id, relay_type)
SELECT DISTINCT
u.id as user_id,
r.id as relay_id,
'key_package' as relay_type
FROM contacts c
JOIN users u ON u.pubkey = c.pubkey
JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
json_each(c.key_package_relays) as relay_value
WHERE json_valid(c.key_package_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Step 4: Extract and insert unique relay URLs from accounts table
-- Extract from accounts.nip65_relays
INSERT OR IGNORE INTO relays (url)
SELECT DISTINCT
json_extract(relay_value.value, '$') as url
FROM accounts,
json_each(accounts.nip65_relays) as relay_value
WHERE json_valid(accounts.nip65_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Extract from accounts.inbox_relays
INSERT OR IGNORE INTO relays (url)
SELECT DISTINCT
json_extract(relay_value.value, '$') as url
FROM accounts,
json_each(accounts.inbox_relays) as relay_value
WHERE json_valid(accounts.inbox_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Extract from accounts.key_package_relays
INSERT OR IGNORE INTO relays (url)
SELECT DISTINCT
json_extract(relay_value.value, '$') as url
FROM accounts,
json_each(accounts.key_package_relays) as relay_value
WHERE json_valid(accounts.key_package_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Step 5: Create user_relays relationships from accounts table
-- Insert accounts.nip65_relays relationships
INSERT OR IGNORE INTO user_relays (user_id, relay_id, relay_type)
SELECT DISTINCT
u.id as user_id,
r.id as relay_id,
'nip65' as relay_type
FROM accounts a
JOIN users u ON u.pubkey = a.pubkey
JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
json_each(a.nip65_relays) as relay_value
WHERE json_valid(a.nip65_relays)
Comment on lines +130 to +132
Copy link
Contributor

@coderabbitai coderabbitai bot Aug 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Same JOIN scoping bug in Step 5; reorder to bring json_each into scope before joining relays.

These three blocks have the same issue; reorder as in Step 3.

Apply these diffs:

--- Step 5: accounts.nip65_relays
-FROM accounts a
-JOIN users u ON u.pubkey = a.pubkey
-JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
-     json_each(a.nip65_relays) as relay_value
+FROM accounts a
+JOIN users u ON u.pubkey = a.pubkey
+CROSS JOIN json_each(a.nip65_relays) AS relay_value
+JOIN relays r ON r.url = json_extract(relay_value.value, '$')
 WHERE json_valid(a.nip65_relays)
--- Step 5: accounts.inbox_relays
-FROM accounts a
-JOIN users u ON u.pubkey = a.pubkey
-JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
-     json_each(a.inbox_relays) as relay_value
+FROM accounts a
+JOIN users u ON u.pubkey = a.pubkey
+CROSS JOIN json_each(a.inbox_relays) AS relay_value
+JOIN relays r ON r.url = json_extract(relay_value.value, '$')
 WHERE json_valid(a.inbox_relays)
--- Step 5: accounts.key_package_relays
-FROM accounts a
-JOIN users u ON u.pubkey = a.pubkey
-JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
-     json_each(a.key_package_relays) as relay_value
+FROM accounts a
+JOIN users u ON u.pubkey = a.pubkey
+CROSS JOIN json_each(a.key_package_relays) AS relay_value
+JOIN relays r ON r.url = json_extract(relay_value.value, '$')
 WHERE json_valid(a.key_package_relays)

Also applies to: 140-142, 154-156

AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Insert accounts.inbox_relays relationships
INSERT OR IGNORE INTO user_relays (user_id, relay_id, relay_type)
SELECT DISTINCT
u.id as user_id,
r.id as relay_id,
'inbox' as relay_type
FROM accounts a
JOIN users u ON u.pubkey = a.pubkey
JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
json_each(a.inbox_relays) as relay_value
WHERE json_valid(a.inbox_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Insert accounts.key_package_relays relationships
INSERT OR IGNORE INTO user_relays (user_id, relay_id, relay_type)
SELECT DISTINCT
u.id as user_id,
r.id as relay_id,
'key_package' as relay_type
FROM accounts a
JOIN users u ON u.pubkey = a.pubkey
JOIN relays r ON r.url = json_extract(relay_value.value, '$'),
json_each(a.key_package_relays) as relay_value
WHERE json_valid(a.key_package_relays)
AND json_extract(relay_value.value, '$') IS NOT NULL
AND json_extract(relay_value.value, '$') != '';

-- Step 6: Migrate accounts to accounts_new table
INSERT INTO accounts_new (pubkey, user_id, settings, last_synced_at)
SELECT
a.pubkey,
u.id as user_id,
a.settings,
CASE
WHEN a.last_synced IS NOT NULL THEN datetime(a.last_synced, 'unixepoch')
ELSE NULL
END as last_synced_at
FROM accounts a
JOIN users u ON u.pubkey = a.pubkey
WHERE NOT EXISTS (SELECT 1 FROM accounts_new WHERE accounts_new.pubkey = a.pubkey);
Comment on lines +164 to +176
Copy link
Contributor

@coderabbitai coderabbitai bot Aug 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

⚠️ Potential issue

Accounts with no matching user are dropped; insert missing users from accounts first.

Currently, accounts_new is populated by joining to users. If a pubkey exists in accounts but not in contacts (Step 1), its account will be skipped. Insert any missing users from accounts before Step 6.

Add this block right after Step 1 (before Step 2):

-- Step 1b: Ensure all account pubkeys exist in users
INSERT INTO users (pubkey, metadata)
SELECT DISTINCT a.pubkey, NULL
FROM accounts a
WHERE a.pubkey IS NOT NULL
  AND a.pubkey != ''
  AND NOT EXISTS (SELECT 1 FROM users u WHERE u.pubkey = a.pubkey);
🤖 Prompt for AI Agents 
In db_migrations/0006_data_migration.sql around lines 160 to 172, accounts rows
with pubkeys that have no matching users are being skipped by the JOIN, so add a
pre-migration step right after Step 1 (before Step 2) that inserts any distinct,
non-null, non-empty pubkeys from the accounts table into users with metadata set
to NULL if they don't already exist; this ensures the subsequent INSERT INTO
accounts_new ... JOIN users will find a user row for every account pubkey and no
accounts are dropped.

16 changes: 8 additions & 8 deletions src/whitenoise/database/accounts_new.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ impl Whitenoise {
) -> Result<Vec<User>, WhitenoiseError> {
let user_rows = sqlx::query_as::<_, UserRow>(
"SELECT u.id, u.pubkey, u.metadata, u.created_at, u.updated_at
FROM account_followers af
JOIN users_new u ON af.user_id = u.id
FROM account_follows af
JOIN users u ON af.user_id = u.id
WHERE af.account_id = ?",
)
.bind(account.id)
Expand Down Expand Up @@ -797,16 +797,16 @@ mod tests {
test_users.push((user_pubkey, metadata.clone()));
}

// Now manually insert the account_followers relationships
// Now manually insert the account_follows relationships
// First we need to get the actual account ID and user IDs from the database
let saved_account = whitenoise.load_account_new(&account_pubkey).await.unwrap();

for (user_pubkey, _) in &test_users {
let saved_user = whitenoise.load_user(user_pubkey).await.unwrap();

// Insert into account_followers table
// Insert into account_follows table
sqlx::query(
"INSERT INTO account_followers (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
"INSERT INTO account_follows (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
)
.bind(saved_account.id)
.bind(saved_user.id)
Expand Down Expand Up @@ -929,7 +929,7 @@ mod tests {

// Insert the follower relationship
sqlx::query(
"INSERT INTO account_followers (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
"INSERT INTO account_follows (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
)
.bind(saved_account.id)
.bind(saved_user.id)
Expand Down Expand Up @@ -1003,7 +1003,7 @@ mod tests {
let saved_user = whitenoise.load_user(&user_pubkey).await.unwrap();

sqlx::query(
"INSERT INTO account_followers (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
"INSERT INTO account_follows (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
)
.bind(saved_account.id)
.bind(saved_user.id)
Expand Down Expand Up @@ -1111,7 +1111,7 @@ mod tests {
let saved_user = whitenoise.load_user(user_pubkey).await.unwrap();

sqlx::query(
"INSERT INTO account_followers (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
"INSERT INTO account_follows (account_id, user_id, created_at, updated_at) VALUES (?, ?, ?, ?)"
)
.bind(saved_account.id)
.bind(saved_user.id)
Expand Down
Loading
Loading