Skip to content

Commit 49c2adb

Browse files
LucienShuiLucienShui
authored
Merge pull request #151 from import-ai/refactor/attachments
refactor(attachments): update attachment URLs
2 parents 3506e2a + 5151ff7 commit 49c2adb

9 files changed

+513
-12
lines changed

‎src/app/app.module.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ import { ApiKeys1754550165406 } from 'omniboxd/migrations/1754550165406-api-keys
4040
import { ResourceAttachments1755059371000 } from 'omniboxd/migrations/1755059371000-resource-attachments';
4141
import { AddTagIdsToResources1755248141570 } from 'omniboxd/migrations/1755248141570-add-tag-ids-to-resources';
4242
import { CleanResourceNames1755396702021 } from 'omniboxd/migrations/1755396702021-clean-resource-names';
43+
import { UpdateAttachmentUrls1755499552000 } from 'omniboxd/migrations/1755499552000-update-attachment-urls';
44+
import { ScanResourceAttachments1755504936756 } from 'omniboxd/migrations/1755504936756-scan-resource-attachments';
4345

4446
@Module({})
4547
export class AppModule implements NestModule {
@@ -110,6 +112,8 @@ export class AppModule implements NestModule {
110112
ResourceAttachments1755059371000,
111113
AddTagIdsToResources1755248141570,
112114
CleanResourceNames1755396702021,
115+
UpdateAttachmentUrls1755499552000,
116+
ScanResourceAttachments1755504936756,
113117
...extraMigrations,
114118
],
115119
migrationsRun: true,

‎src/migrations/1755499552000-update-attachment-urls.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { MigrationInterface, QueryRunner } from 'typeorm';
2+
3+
export class UpdateAttachmentUrls1755499552000 implements MigrationInterface {
4+
public async up(queryRunner: QueryRunner): Promise<void> {
5+
const batchSize = 100;
6+
let offset = 0;
7+
8+
while (true) {
9+
const resources = await queryRunner.query(
10+
`
11+
SELECT id, content
12+
FROM resources
13+
WHERE content != '' AND deleted_at IS NULL
14+
ORDER BY id
15+
LIMIT $1 OFFSET $2
16+
`,
17+
[batchSize, offset],
18+
);
19+
20+
if (resources.length === 0) {
21+
break;
22+
}
23+
24+
for (const resource of resources) {
25+
const originalContent = resource.content;
26+
27+
// Replace /api/v1/attachments/images/{id} and
28+
// /api/v1/attachments/media/{id} with attachments/{id}
29+
const updatedContent = originalContent.replace(
30+
/\/api\/v1\/attachments\/(images|media)\/([a-zA-Z0-9\._-]+)/g,
31+
'attachments/$2',
32+
);
33+
34+
// Only update if content actually changed
35+
if (updatedContent !== originalContent) {
36+
await queryRunner.query(
37+
`UPDATE resources SET content = $1 WHERE id = $2`,
38+
[updatedContent, resource.id],
39+
);
40+
}
41+
}
42+
43+
offset += batchSize;
44+
45+
// If we got fewer resources than batch size, we're done
46+
if (resources.length < batchSize) {
47+
break;
48+
}
49+
}
50+
}
51+
52+
public down(): Promise<void> {
53+
throw new Error('Not supported.');
54+
}
55+
}

‎src/migrations/1755504936756-scan-resource-attachments.ts‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import { MigrationInterface, QueryRunner } from 'typeorm';
2+
3+
export class ScanResourceAttachments1755504936756
4+
implements MigrationInterface
5+
{
6+
public async up(queryRunner: QueryRunner): Promise<void> {
7+
const batchSize = 100;
8+
let offset = 0;
9+
10+
while (true) {
11+
const resources = await queryRunner.query(
12+
`
13+
SELECT id, namespace_id, content
14+
FROM resources
15+
WHERE content != '' AND deleted_at IS NULL
16+
ORDER BY id
17+
LIMIT $1 OFFSET $2
18+
`,
19+
[batchSize, offset],
20+
);
21+
22+
if (resources.length === 0) {
23+
break;
24+
}
25+
26+
for (const resource of resources) {
27+
const content = resource.content;
28+
29+
// Find all attachment references in the format: attachments/{id}
30+
const attachmentMatches = content.match(
31+
/attachments\/([a-zA-Z0-9\._-]+)/g,
32+
);
33+
34+
for (const match of attachmentMatches || []) {
35+
// Extract attachment ID from the match (remove "attachments/" prefix only)
36+
const attachmentId = match.replace('attachments/', '');
37+
38+
// Check if this resource-attachment relation already exists
39+
const existingRelation = await queryRunner.query(
40+
`
41+
SELECT id FROM resource_attachments
42+
WHERE namespace_id = $1 AND resource_id = $2 AND attachment_id = $3 AND deleted_at IS NULL
43+
`,
44+
[resource.namespace_id, resource.id, attachmentId],
45+
);
46+
47+
// Only create the relation if it doesn't exist
48+
if (existingRelation.length === 0) {
49+
await queryRunner.query(
50+
`
51+
INSERT INTO resource_attachments (namespace_id, resource_id, attachment_id)
52+
VALUES ($1, $2, $3)
53+
`,
54+
[resource.namespace_id, resource.id, attachmentId],
55+
);
56+
}
57+
}
58+
}
59+
60+
offset += batchSize;
61+
62+
// If we got fewer resources than batch size, we're done
63+
if (resources.length < batchSize) {
64+
break;
65+
}
66+
}
67+
}
68+
69+
public down(): Promise<void> {
70+
throw new Error('Not supported.');
71+
}
72+
}

‎src/migrations/clean-resource-names.e2e-spec.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ describe('CleanResourceNames Migration E2E', () => {
5656
it('should decode URL-encoded resource names', async () => {
5757
// Setup: Insert URL-encoded names
5858
await queryRunner.query(`
59-
INSERT INTO resources (id, name) VALUES
59+
INSERT INTO resources (id, name) VALUES
6060
('res1', 'Hello%20World'),
6161
('res2', 'My%20Document%2Etxt'),
6262
('res3', 'Normal Name'),
@@ -82,7 +82,7 @@ describe('CleanResourceNames Migration E2E', () => {
8282

8383
it('should handle double-encoded names', async () => {
8484
await queryRunner.query(`
85-
INSERT INTO resources (id, name) VALUES
85+
INSERT INTO resources (id, name) VALUES
8686
('res1', 'Hello%2520World')
8787
`);
8888

@@ -104,7 +104,7 @@ describe('CleanResourceNames Migration E2E', () => {
104104

105105
await queryRunner.query(
106106
`
107-
INSERT INTO resources (id, name) VALUES
107+
INSERT INTO resources (id, name) VALUES
108108
('res1', $1),
109109
('res2', 'Normal Text')
110110
`,
@@ -134,7 +134,7 @@ describe('CleanResourceNames Migration E2E', () => {
134134

135135
await queryRunner.query(
136136
`
137-
INSERT INTO resources (id, name) VALUES
137+
INSERT INTO resources (id, name) VALUES
138138
('res1', $1)
139139
`,
140140
[urlEncodedMojibake],
@@ -154,7 +154,7 @@ describe('CleanResourceNames Migration E2E', () => {
154154
describe('Edge cases', () => {
155155
it('should not modify already clean names', async () => {
156156
await queryRunner.query(`
157-
INSERT INTO resources (id, name) VALUES
157+
INSERT INTO resources (id, name) VALUES
158158
('res1', 'Clean Name'),
159159
('res2', 'Another Clean Name 123'),
160160
('res3', 'With-Special_Chars.txt'),
@@ -176,7 +176,7 @@ describe('CleanResourceNames Migration E2E', () => {
176176

177177
it('should handle empty and null names gracefully', async () => {
178178
await queryRunner.query(`
179-
INSERT INTO resources (id, name) VALUES
179+
INSERT INTO resources (id, name) VALUES
180180
('res1', ''),
181181
('res2', 'Valid Name')
182182
`);

‎src/migrations/scan-resource-attachments.e2e-spec.ts‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
import { DataSource, QueryRunner } from 'typeorm';
2+
import { ScanResourceAttachments1755504936756 } from './1755504936756-scan-resource-attachments';
3+
4+
// Import all prior migrations in chronological order
5+
import { Init1751900000000 } from './1751900000000-init';
6+
import { UserOptions1751904560034 } from './1751904560034-user-options';
7+
import { Tags1751905414493 } from './1751905414493-tags';
8+
import { UserBindings1752652489640 } from './1752652489640-user-bindings.ts';
9+
import { NullUserEmail1752814358259 } from './1752814358259-null-user-email';
10+
import { Shares1753866547335 } from './1753866547335-shares';
11+
import { ApiKeys1754550165406 } from './1754550165406-api-keys';
12+
import { ResourceAttachments1755059371000 } from './1755059371000-resource-attachments';
13+
import { AddTagIdsToResources1755248141570 } from './1755248141570-add-tag-ids-to-resources';
14+
import { CleanResourceNames1755396702021 } from './1755396702021-clean-resource-names';
15+
import { UpdateAttachmentUrls1755499552000 } from './1755499552000-update-attachment-urls';
16+
17+
describe('ScanResourceAttachments Migration E2E', () => {
18+
let dataSource: DataSource;
19+
let queryRunner: QueryRunner;
20+
21+
beforeAll(async () => {
22+
dataSource = new DataSource({
23+
type: 'postgres',
24+
url: process.env.OBB_POSTGRES_URL,
25+
entities: [],
26+
migrations: [
27+
Init1751900000000,
28+
UserOptions1751904560034,
29+
Tags1751905414493,
30+
UserBindings1752652489640,
31+
NullUserEmail1752814358259,
32+
Shares1753866547335,
33+
ApiKeys1754550165406,
34+
ResourceAttachments1755059371000,
35+
AddTagIdsToResources1755248141570,
36+
CleanResourceNames1755396702021,
37+
UpdateAttachmentUrls1755499552000,
38+
],
39+
synchronize: false,
40+
migrationsRun: false,
41+
});
42+
await dataSource.initialize();
43+
44+
// Enable UUID extension
45+
await dataSource.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"');
46+
47+
// Run migrations manually
48+
await dataSource.runMigrations();
49+
});
50+
51+
beforeEach(async () => {
52+
queryRunner = dataSource.createQueryRunner();
53+
await queryRunner.connect();
54+
await queryRunner.startTransaction();
55+
56+
// Create a test namespace specific to this migration test
57+
await queryRunner.query(`
58+
INSERT INTO namespaces (id, name) VALUES ('scan-resource-attachments-test', 'ScanResourceAttachments Migration Test')
59+
ON CONFLICT (id) DO NOTHING
60+
`);
61+
});
62+
63+
afterEach(async () => {
64+
await queryRunner.rollbackTransaction();
65+
await queryRunner.release();
66+
});
67+
68+
afterAll(async () => {
69+
await dataSource.destroy();
70+
});
71+
72+
describe('Attachment relation creation', () => {
73+
it('should create resource-attachment relations for single attachment', async () => {
74+
// Setup: Insert resource with attachment reference
75+
await queryRunner.query(`
76+
INSERT INTO resources (id, name, namespace_id, resource_type, content) VALUES
77+
('res1', 'Test Document', 'scan-resource-attachments-test', 'doc', 'Here is an image: attachments/abc123.jpg')
78+
`);
79+
80+
// Execute migration
81+
const migration = new ScanResourceAttachments1755504936756();
82+
await migration.up(queryRunner);
83+
84+
// Verify resource_attachments table entry was created for our specific resource
85+
const relations = await queryRunner.query(`
86+
SELECT namespace_id, resource_id, attachment_id FROM resource_attachments
87+
WHERE namespace_id = 'scan-resource-attachments-test' AND resource_id = 'res1' AND attachment_id = 'abc123.jpg'
88+
ORDER BY attachment_id
89+
`);
90+
91+
expect(relations).toHaveLength(1);
92+
expect(relations[0]).toEqual({
93+
namespace_id: 'scan-resource-attachments-test',
94+
resource_id: 'res1',
95+
attachment_id: 'abc123.jpg',
96+
});
97+
});
98+
99+
it('should create multiple resource-attachment relations for multiple attachments', async () => {
100+
// Setup: Insert resource with multiple attachment references
101+
await queryRunner.query(`
102+
INSERT INTO resources (id, name, namespace_id, resource_type, content) VALUES
103+
('res1', 'Multi-attachment Doc', 'scan-resource-attachments-test', 'doc',
104+
'Images: attachments/photo1.png and attachments/photo2.gif
105+
Audio: attachments/sound.wav
106+
Video: attachments/clip.mp4')
107+
`);
108+
109+
// Execute migration
110+
const migration = new ScanResourceAttachments1755504936756();
111+
await migration.up(queryRunner);
112+
113+
// Verify all relations were created
114+
const relations = await queryRunner.query(`
115+
SELECT attachment_id FROM resource_attachments
116+
WHERE namespace_id = 'scan-resource-attachments-test' AND resource_id = 'res1'
117+
ORDER BY attachment_id
118+
`);
119+
120+
expect(relations.map((r) => r.attachment_id)).toEqual([
121+
'clip.mp4',
122+
'photo1.png',
123+
'photo2.gif',
124+
'sound.wav',
125+
]);
126+
});
127+
128+
it('should handle various file extensions correctly', async () => {
129+
// Setup: Insert resource with different file types
130+
await queryRunner.query(`
131+
INSERT INTO resources (id, name, namespace_id, resource_type, content) VALUES
132+
('res1', 'Various Extensions', 'scan-resource-attachments-test', 'doc',
133+
'Files: attachments/image.jpeg attachments/doc.pdf attachments/audio.mp3
134+
attachments/video.webm attachments/data.json attachments/archive.zip
135+
attachments/drawing.svg attachments/presentation.pptx')
136+
`);
137+
138+
// Execute migration
139+
const migration = new ScanResourceAttachments1755504936756();
140+
await migration.up(queryRunner);
141+
142+
// Verify all relations were created
143+
const relations = await queryRunner.query(`
144+
SELECT attachment_id FROM resource_attachments
145+
WHERE namespace_id = 'scan-resource-attachments-test' AND resource_id = 'res1'
146+
ORDER BY attachment_id
147+
`);
148+
149+
expect(relations.map((r) => r.attachment_id)).toEqual([
150+
'archive.zip',
151+
'audio.mp3',
152+
'data.json',
153+
'doc.pdf',
154+
'drawing.svg',
155+
'image.jpeg',
156+
'presentation.pptx',
157+
'video.webm',
158+
]);
159+
});
160+
161+
it('should handle complex filenames with underscores and hyphens', async () => {
162+
// Setup: Insert resource with complex attachment filenames
163+
await queryRunner.query(`
164+
INSERT INTO resources (id, name, namespace_id, resource_type, content) VALUES
165+
('res1', 'Complex Filenames', 'scan-resource-attachments-test', 'doc',
166+
'Files: attachments/file-with-hyphens.png attachments/file_with_underscores.jpg
167+
attachments/MiXeD_CaSe-File123.jpeg attachments/very-long_filename-with-123_numbers.webm')
168+
`);
169+
170+
// Execute migration
171+
const migration = new ScanResourceAttachments1755504936756();
172+
await migration.up(queryRunner);
173+
174+
// Verify all relations were created
175+
const relations = await queryRunner.query(`
176+
SELECT attachment_id FROM resource_attachments
177+
WHERE namespace_id = 'scan-resource-attachments-test' AND resource_id = 'res1'
178+
ORDER BY attachment_id
179+
`);
180+
181+
expect(relations.map((r) => r.attachment_id)).toEqual([
182+
'file-with-hyphens.png',
183+
'file_with_underscores.jpg',
184+
'MiXeD_CaSe-File123.jpeg',
185+
'very-long_filename-with-123_numbers.webm',
186+
]);
187+
});
188+
});
189+
});

0 commit comments

Comments
 (0)