Skip to content

Commit 26bea3c

Browse files
raideerraideer
committed
chore: indexer refactoring
1 parent 67c0dee commit 26bea3c

File tree

8 files changed

+62
-53
lines changed

8 files changed

+62
-53
lines changed

‎package.json

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,6 @@
6767
"type": "boolean",
6868
"default": true,
6969
"description": "Enable hover decorations for Magento 2 XML files."
70-
},
71-
"magento-toolbox.indexingBatchSize": {
72-
"type": "number",
73-
"default": 25,
74-
"description": "Number of files to index in a single batch."
7570
}
7671
}
7772
},

‎src/extension.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ export async function activate(context: vscode.ExtensionContext) {
3737
Object.values(commands).forEach(command => {
3838
const instance = new command() as Command;
3939

40-
Logger.log('Registering command', instance.getCommand());
40+
Logger.log('EXT', 'Registering command', instance.getCommand());
4141

4242
const disposable = vscode.commands.registerCommand(instance.getCommand(), async (...args) => {
4343
try {

‎src/generator/HandlebarsTemplateRenderer.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ export default class HandlebarsTemplateRenderer {
4040
const content = compiledTemplate(data);
4141
return content;
4242
} catch (error) {
43-
Logger.log('Failed to generate template', String(error));
43+
Logger.log('RENDER', 'Failed to generate template', String(error));
4444
throw error;
4545
}
4646
}

‎src/indexer/IndexManager.ts

Lines changed: 43 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ type IndexerDataMap = {
4242
};
4343

4444
class IndexManager {
45+
private static readonly INDEXING_BATCH_SIZE = 50;
4546
private static readonly REPORTING_BATCH_SIZE = 500;
4647

4748
protected indexers: IndexerInstance[] = [];
@@ -75,100 +76,107 @@ class IndexManager {
7576
): Promise<void> {
7677
const workspaceUri = workspaceFolder.uri;
7778

78-
Logger.logWithTime('Indexing workspace', workspaceFolder.name);
79-
8079
for (const indexer of this.indexers) {
8180
if (!force) {
82-
Logger.logWithTime('Loading index from storage', workspaceFolder.name, indexer.getId());
81+
Logger.log('INDEXER', 'Loading index from storage', workspaceFolder.name, indexer.getId());
8382
this.indexStorage.loadIndex(workspaceFolder, indexer.getId(), indexer.getVersion());
8483

8584
if (!this.shouldIndex(workspaceFolder, indexer)) {
86-
Logger.logWithTime('Loaded index from storage', workspaceFolder.name, indexer.getId());
85+
Logger.log('INDEXER', 'Loaded index from storage', workspaceFolder.name, indexer.getId());
8786
continue;
8887
}
8988
}
90-
Logger.logWithTime('Indexing', indexer.getName());
89+
90+
const startIndexing = Date.now();
91+
Logger.log('INDEXER', 'Indexing', indexer.getName());
9192

9293
progress.report({
9394
message: `Indexing - ${indexer.getName()} [...]`,
9495
});
9596

9697
const indexData = new Map();
9798

98-
Logger.logWithTime('Discovering files to index');
99+
const startDiscover = Date.now();
100+
Logger.log('INDEXER', 'Discovering files to index');
99101

100102
const files = await workspace.findFiles(
101103
indexer.getPattern(workspaceUri),
102104
indexer.getExcludePattern(workspaceUri)
103105
);
104106

107+
const discoverDuration = Date.now() - startDiscover;
108+
Logger.log('INDEXER', 'Found', files.length, 'files to index in', discoverDuration, 'ms');
109+
105110
let doneCount = 0;
106111
const totalCount = files.length;
107-
Logger.logWithTime('Found', totalCount, 'files to index');
108112

109-
const batchSize = Config.get<number>('magento-toolbox.indexingBatchSize', 25);
113+
const startIndex = Date.now();
114+
115+
for (let i = 0; i < files.length; i += IndexManager.INDEXING_BATCH_SIZE) {
116+
const batch = files.slice(i, i + IndexManager.INDEXING_BATCH_SIZE);
110117

111-
for (let i = 0; i < files.length; i += batchSize) {
112-
const batch = files.slice(i, i + batchSize);
118+
const promises = batch.map(async file => {
119+
try {
120+
const data = await indexer.indexFile(file);
113121

114-
await Promise.all(
115-
batch.map(async file => {
116-
try {
117-
const data = await indexer.indexFile(file);
122+
if (data !== undefined) {
123+
indexData.set(file.fsPath, data);
124+
}
125+
} catch (error) {
126+
Logger.error('Error indexing file', file.fsPath, String(error));
127+
}
128+
});
118129

119-
if (data !== undefined) {
120-
indexData.set(file.fsPath, data);
121-
}
130+
await Promise.all(promises);
122131

123-
doneCount++;
132+
doneCount += batch.length;
124133

125-
if (doneCount % IndexManager.REPORTING_BATCH_SIZE === 0) {
126-
Logger.logWithTime('Indexed', doneCount, 'files of', totalCount);
134+
if (doneCount % IndexManager.REPORTING_BATCH_SIZE === 0) {
135+
Logger.log('Indexed', doneCount, 'files of', totalCount);
127136

128-
progress.report({
129-
message: `Indexing - ${indexer.getName()} [${doneCount}/${totalCount}]`,
130-
});
131-
}
132-
} catch (error) {
133-
Logger.error('Error indexing file', file.fsPath, String(error));
134-
}
135-
})
136-
);
137+
progress.report({
138+
message: `Indexing - ${indexer.getName()} [${doneCount}/${totalCount}]`,
139+
});
140+
}
137141
}
138142

143+
const indexDuration = Date.now() - startIndex;
144+
Logger.log('INDEXER', `Indexed ${doneCount} files of ${totalCount} in ${indexDuration}ms`);
145+
139146
this.indexStorage.set(workspaceFolder, indexer.getId(), indexData);
140147
this.indexStorage.saveIndex(workspaceFolder, indexer.getId(), indexer.getVersion());
141148

142149
clear([indexer.getId()]);
143150

144-
Logger.logWithTime('Indexing', indexer.getName(), 'done');
151+
const indexingDuration = Date.now() - startIndexing;
152+
Logger.log('INDEXER', 'Indexing', indexer.getName(), 'done in', indexingDuration, 'ms');
145153

146154
progress.report({ message: `Indexing - ${indexer.getName()} [done]` });
147155
}
148156

149-
Logger.logWithTime('Finished indexing workspace', workspaceFolder.name);
157+
Logger.log('INDEXER', 'Finished indexing workspace', workspaceFolder.name);
150158
}
151159

152160
public async indexFile(workspaceFolder: WorkspaceFolder, file: Uri): Promise<void> {
153-
Logger.logWithTime('Indexing file', file.fsPath);
161+
Logger.log('INDEXER', 'Indexing file', file.fsPath);
154162

155163
await Promise.all(
156164
this.indexers.map(async indexer => {
157165
await this.indexFileInner(workspaceFolder, file, indexer);
158166
})
159167
);
160168

161-
Logger.logWithTime('Finished indexing file', file.fsPath);
169+
Logger.log('INDEXER', 'Finished indexing file', file.fsPath);
162170
}
163171

164172
public async indexFiles(workspaceFolder: WorkspaceFolder, files: Uri[]): Promise<void> {
165-
Logger.logWithTime(`Indexing ${files.length} files`);
173+
Logger.log('INDEXER', `Indexing ${files.length} files`);
166174

167175
for (const indexer of this.indexers) {
168176
await Promise.all(files.map(file => this.indexFileInner(workspaceFolder, file, indexer)));
169177
}
170178

171-
Logger.logWithTime(`Finished indexing ${files.length} files`);
179+
Logger.log('INDEXER', `Finished indexing ${files.length} files`);
172180
}
173181

174182
public getIndexStorageData<T = any>(

‎src/indexer/Indexer.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ export abstract class Indexer<D = any> {
77
public abstract getPattern(uri: Uri): GlobPattern;
88

99
public getExcludePattern(uri: Uri): GlobPattern | null {
10-
return new RelativePattern(uri, '{**/node_modules/**,**/dev/**,**/pub/**}');
10+
return null;
1111
}
1212

1313
public abstract indexFile(uri: Uri): Promise<D | undefined>;

‎src/indexer/autoload-namespace/AutoloadNamespaceIndexer.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,10 @@ export default class AutoloadNamespaceIndexer extends Indexer<Namespace[]> {
7575
relativeBaseDirectory: string
7676
): Promise<Namespace[]> {
7777
const baseDirectoryUri = Uri.joinPath(baseDirectory, relativeBaseDirectory.replace(/\\$/, ''));
78-
const files = await FileSystem.readDirectoryRecursive(baseDirectoryUri);
78+
const files = await FileSystem.readDirectoryRecursive(
79+
baseDirectoryUri,
80+
new RelativePattern(baseDirectoryUri, '**/*.php')
81+
);
7982

8083
return files
8184
.filter(file => file.endsWith('.php'))

‎src/util/FileSystem.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,11 @@ export default class FileSystem {
2525
return files.map(([name]) => name);
2626
}
2727

28-
public static async readDirectoryRecursive(uri: Uri): Promise<string[]> {
29-
const files = await workspace.findFiles(new RelativePattern(uri, '**/*.php'));
28+
public static async readDirectoryRecursive(
29+
uri: Uri,
30+
pattern: RelativePattern
31+
): Promise<string[]> {
32+
const files = await workspace.findFiles(pattern);
3033
return files.map(file => path.relative(uri.fsPath, file.fsPath));
3134
}
3235

‎src/util/Logger.ts

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@ import { window } from 'vscode';
33
export class Logger {
44
private channel = window.createOutputChannel('Magento Toolbox');
55

6-
public log(...message: (string | number)[]) {
7-
this.channel.appendLine(message.join(' '));
8-
console.log('[Magento Toolbox]', ...message);
6+
public log(category: string, ...message: (string | number)[]) {
7+
const text = [`[${category}]`, ...message].join(' ');
8+
this.channel.appendLine(text);
99
}
1010

11-
public logWithTime(...message: (string | number)[]) {
12-
this.log(new Date().toISOString(), ...message);
11+
public logWithTime(category: string, ...message: (string | number)[]) {
12+
this.log(category, new Date().toISOString(), ...message);
1313
}
1414

15-
public error(...message: (string | number)[]) {
16-
this.logWithTime('ERROR', ...message);
15+
public error(category: string, ...message: (string | number)[]) {
16+
this.log(category, 'ERROR', ...message);
1717
}
1818
}
1919

0 commit comments

Comments
 (0)