Skip to content

Commit f6eb979

Browse files
sshadersshader
committed
Add extraction of analyzed functions + generation of TS api and OpenAPI spec
1 parent 362f03a commit f6eb979

File tree

12 files changed

+1300
-2
lines changed

12 files changed

+1300
-2
lines changed

‎analyze.mts‎

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
import { execSync } from 'child_process'
2+
import { JSONValue } from 'convex/values'
3+
import fs from 'fs'
4+
import path from 'path'
5+
6+
/*
7+
Usage:
8+
npx ts-node-esm analyze.mts convex analyze
9+
10+
Assumes there's already `convex/analyze` with a `helpers.ts` file
11+
Outputs content in /tmp/analyzeResult
12+
*/
13+
14+
type Visibility = { kind: 'public' } | { kind: 'internal' }
15+
16+
type UdfType = 'action' | 'mutation' | 'query' | 'httpAction'
17+
18+
export type AnalyzedFunctions = Array<{
19+
name: string
20+
udfType: UdfType
21+
visibility: Visibility | null
22+
args: JSONValue | null
23+
}>
24+
25+
async function analyzeModule(filePath: string): Promise<AnalyzedFunctions> {
26+
const importedModule = await import(filePath)
27+
28+
const functions: Map<
29+
string,
30+
{
31+
udfType: UdfType
32+
visibility: Visibility | null
33+
args: JSONValue | null
34+
}
35+
> = new Map()
36+
for (const [name, value] of Object.entries(importedModule)) {
37+
if (value === undefined || value === null) {
38+
continue
39+
}
40+
41+
let udfType: UdfType
42+
if (
43+
Object.prototype.hasOwnProperty.call(value, 'isAction') &&
44+
Object.prototype.hasOwnProperty.call(value, 'invokeAction')
45+
) {
46+
udfType = 'action'
47+
} else if (
48+
Object.prototype.hasOwnProperty.call(value, 'isQuery') &&
49+
Object.prototype.hasOwnProperty.call(value, 'invokeQuery')
50+
) {
51+
udfType = 'query'
52+
} else if (
53+
Object.prototype.hasOwnProperty.call(value, 'isMutation') &&
54+
Object.prototype.hasOwnProperty.call(value, 'invokeMutation')
55+
) {
56+
udfType = 'mutation'
57+
} else if (
58+
Object.prototype.hasOwnProperty.call(value, 'isHttp') &&
59+
(Object.prototype.hasOwnProperty.call(value, 'invokeHttpEndpoint') ||
60+
Object.prototype.hasOwnProperty.call(value, 'invokeHttpAction'))
61+
) {
62+
udfType = 'httpAction'
63+
} else {
64+
continue
65+
}
66+
const isPublic = Object.prototype.hasOwnProperty.call(value, 'isPublic')
67+
const isInternal = Object.prototype.hasOwnProperty.call(value, 'isInternal')
68+
69+
let args: string | null = null
70+
if (
71+
Object.prototype.hasOwnProperty.call(value, 'exportArgs') &&
72+
typeof (value as any).exportArgs === 'function'
73+
) {
74+
const exportedArgs = (value as any).exportArgs()
75+
if (typeof exportedArgs === 'string') {
76+
args = JSON.parse(exportedArgs)
77+
}
78+
}
79+
80+
if (isPublic && isInternal) {
81+
console.debug(
82+
`Skipping function marked as both public and internal: ${name}`
83+
)
84+
continue
85+
} else if (isPublic) {
86+
functions.set(name, { udfType, visibility: { kind: 'public' }, args })
87+
} else if (isInternal) {
88+
functions.set(name, {
89+
udfType,
90+
visibility: { kind: 'internal' },
91+
args,
92+
})
93+
} else {
94+
functions.set(name, { udfType, visibility: null, args })
95+
}
96+
}
97+
const analyzed = [...functions.entries()].map(([name, properties]) => {
98+
// Finding line numbers is best effort. We should return the analyzed
99+
// function even if we fail to find the exact line number.
100+
return {
101+
name,
102+
...properties,
103+
}
104+
})
105+
106+
return analyzed
107+
}
108+
109+
// Returns a generator of { isDir, path } for all paths
110+
// within dirPath in some topological order (not including
111+
// dirPath itself).
112+
export function* walkDir(
113+
dirPath: string
114+
): Generator<{ isDir: boolean; path: string }, void, void> {
115+
for (const dirEntry of fs
116+
.readdirSync(dirPath, { withFileTypes: true })
117+
.sort()) {
118+
const childPath = path.join(dirPath, dirEntry.name)
119+
if (dirEntry.isDirectory()) {
120+
yield { isDir: true, path: childPath }
121+
yield* walkDir(childPath)
122+
} else if (dirEntry.isFile()) {
123+
yield { isDir: false, path: childPath }
124+
}
125+
}
126+
}
127+
export async function entryPoints(
128+
dir: string,
129+
verbose: boolean
130+
): Promise<string[]> {
131+
const entryPoints = []
132+
133+
const log = (line: string) => {
134+
if (verbose) {
135+
console.log(line)
136+
}
137+
}
138+
139+
for (const { isDir, path: fpath } of walkDir(dir)) {
140+
if (isDir) {
141+
continue
142+
}
143+
const relPath = path.relative(dir, fpath)
144+
const base = path.parse(fpath).base
145+
146+
if (relPath.startsWith('_deps' + path.sep)) {
147+
throw new Error(
148+
`The path "${fpath}" is within the "_deps" directory, which is reserved for dependencies. Please move your code to another directory.`
149+
)
150+
} else if (relPath.startsWith('_generated' + path.sep)) {
151+
log(`Skipping ${fpath}`)
152+
} else if (base.startsWith('.')) {
153+
log(`Skipping dotfile ${fpath}`)
154+
} else if (base === 'README.md') {
155+
log(`Skipping ${fpath}`)
156+
} else if (base === '_generated.ts') {
157+
log(`Skipping ${fpath}`)
158+
} else if (base === 'schema.ts') {
159+
log(`Skipping ${fpath}`)
160+
} else if ((base.match(/\./g) || []).length > 1) {
161+
log(`Skipping ${fpath} that contains multiple dots`)
162+
} else if (base === 'tsconfig.json') {
163+
log(`Skipping ${fpath}`)
164+
} else if (relPath.endsWith('.config.js')) {
165+
log(`Skipping ${fpath}`)
166+
} else if (relPath.includes(' ')) {
167+
log(`Skipping ${relPath} because it contains a space`)
168+
} else if (base.endsWith('.d.ts')) {
169+
log(`Skipping ${fpath} declaration file`)
170+
} else if (base.endsWith('.json')) {
171+
log(`Skipping ${fpath} json file`)
172+
} else {
173+
log(`Preparing ${fpath}`)
174+
entryPoints.push(fpath)
175+
}
176+
}
177+
178+
// If using TypeScript, require that at least one line starts with `export` or `import`,
179+
// a TypeScript requirement. This prevents confusing type errors described in CX-5067.
180+
const nonEmptyEntryPoints = entryPoints.filter((fpath) => {
181+
// This check only makes sense for TypeScript files
182+
if (!fpath.endsWith('.ts') && !fpath.endsWith('.tsx')) {
183+
return true
184+
}
185+
const contents = fs.readFileSync(fpath, { encoding: 'utf-8' })
186+
if (/^\s{0,100}(import|export)/m.test(contents)) {
187+
return true
188+
}
189+
log(
190+
`Skipping ${fpath} because it has no export or import to make it a valid TypeScript module`
191+
)
192+
})
193+
194+
return nonEmptyEntryPoints
195+
}
196+
197+
export type CanonicalizedModulePath = string
198+
199+
export async function analyze(
200+
convexDir: string
201+
): Promise<Record<CanonicalizedModulePath, AnalyzedFunctions>> {
202+
const modules: Record<CanonicalizedModulePath, AnalyzedFunctions> = {}
203+
const files = await entryPoints(convexDir, false)
204+
for (const modulePath of files) {
205+
const filePath = path.join(convexDir, modulePath)
206+
modules[modulePath] = await analyzeModule(filePath)
207+
}
208+
return modules
209+
}
210+
211+
export function importPath(modulePath: string) {
212+
// Replace backslashes with forward slashes.
213+
const filePath = modulePath.replace(/\\/g, '/')
214+
// Strip off the file extension.
215+
const lastDot = filePath.lastIndexOf('.')
216+
return filePath.slice(0, lastDot === -1 ? undefined : lastDot)
217+
}
218+
219+
function generateFile(paths: string[], filename: string, isNode: boolean) {
220+
const imports: string[] = []
221+
const moduleGroupKeys: string[] = []
222+
for (const p of paths) {
223+
const safeModulePath = importPath(p).replace(/\//g, '_').replace(/-/g, '_')
224+
imports.push(`import * as ${safeModulePath} from "../${p}";`)
225+
moduleGroupKeys.push(`"${p}": ${safeModulePath},`)
226+
}
227+
228+
const content = `
229+
${isNode ? '"use node";' : ''}
230+
import { internalAction } from "../_generated/server.js";
231+
import { analyzeModuleGroups } from "./helpers";
232+
${imports.join('\n')}
233+
export default internalAction((ctx) => {
234+
return analyzeModuleGroups({
235+
${moduleGroupKeys.join('\n')}
236+
})
237+
})
238+
`
239+
fs.writeFileSync(filename, content)
240+
}
241+
242+
async function main(convexDir: string, analyzeDir: string) {
243+
// analyzeDir is nested under convexDir and should contain a
244+
// `helpers.ts` with a `analyzeModuleGroups` function
245+
246+
// TODO: clear out analyzeDir
247+
248+
// Get a list of modules split by module type
249+
execSync('rm -rf /tmp/debug_bundle_path')
250+
execSync('npx convex dev --once --debug-bundle-path /tmp/debug_bundle_path')
251+
const outputStr = fs.readFileSync('/tmp/debug_bundle_path/fullConfig.json', {
252+
encoding: 'utf-8',
253+
})
254+
const output = JSON.parse(outputStr)
255+
if (!fs.existsSync('/tmp/debugConvexDir')) {
256+
fs.mkdirSync('/tmp/debugConvexDir')
257+
}
258+
const isolatePaths: string[] = []
259+
const nodePaths: string[] = []
260+
for (const m of output.modules) {
261+
if (m.path.startsWith('_deps')) {
262+
continue
263+
}
264+
if (m.path.startsWith(analyzeDir)) {
265+
continue
266+
}
267+
if (m.path === 'schema.js') {
268+
continue
269+
}
270+
if (m.path === 'auth.config.js') {
271+
continue
272+
}
273+
if (m.environment === 'isolate') {
274+
isolatePaths.push(m.path)
275+
} else {
276+
nodePaths.push(m.path)
277+
}
278+
}
279+
280+
// Split these into chunks
281+
const chunkSize = 10
282+
let chunkNumber = 0
283+
// Generate files in the analyze directory for each of these
284+
for (let i = 0; i < isolatePaths.length; i += chunkSize) {
285+
const chunk = isolatePaths.slice(i, i + chunkSize)
286+
generateFile(
287+
chunk,
288+
`${convexDir}/${analyzeDir}/group${chunkNumber}.ts`,
289+
false
290+
)
291+
chunkNumber += 1
292+
}
293+
for (let i = 0; i < nodePaths.length; i += chunkSize) {
294+
const chunk = nodePaths.slice(i, i + chunkSize)
295+
generateFile(
296+
chunk,
297+
`${convexDir}/${analyzeDir}/group${chunkNumber}.ts`,
298+
true
299+
)
300+
chunkNumber += 1
301+
}
302+
303+
// Push our generated functions to dev
304+
execSync('npx convex dev --once')
305+
306+
// Run all the functions and collect the result
307+
let fullResults: Record<string, any> = {}
308+
for (let i = 0; i < chunkNumber; i += 1) {
309+
const result = execSync(`npx convex run ${analyzeDir}/group${i}:default`, {
310+
maxBuffer: 2 ** 30,
311+
}).toString()
312+
console.log(result)
313+
fullResults = {
314+
...fullResults,
315+
...JSON.parse(result),
316+
}
317+
}
318+
fs.writeFileSync('/tmp/analyzeResult', JSON.stringify(fullResults, null, 2))
319+
console.log('Result written to /tmp/analyzeResult')
320+
}
321+
322+
await main(process.argv[2], process.argv[3])

‎convex/_generated/api.d.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,16 @@ import type {
1414
FilterApi,
1515
FunctionReference,
1616
} from "convex/server";
17+
import type * as analyze_group0 from "../analyze/group0.js";
18+
import type * as analyze_group1 from "../analyze/group1.js";
19+
import type * as analyze_group2 from "../analyze/group2.js";
20+
import type * as analyze_group3 from "../analyze/group3.js";
21+
import type * as analyze_helpers from "../analyze/helpers.js";
1722
import type * as cards from "../cards.js";
1823
import type * as dealCards from "../dealCards.js";
1924
import type * as functions from "../functions.js";
2025
import type * as games from "../games.js";
26+
import type * as http from "../http.js";
2127
import type * as lib_functions from "../lib/functions.js";
2228
import type * as lib_middlewareUtils from "../lib/middlewareUtils.js";
2329
import type * as lib_validators from "../lib/validators.js";
@@ -44,10 +50,16 @@ import type * as users from "../users.js";
4450
* ```
4551
*/
4652
declare const fullApi: ApiFromModules<{
53+
"analyze/group0": typeof analyze_group0;
54+
"analyze/group1": typeof analyze_group1;
55+
"analyze/group2": typeof analyze_group2;
56+
"analyze/group3": typeof analyze_group3;
57+
"analyze/helpers": typeof analyze_helpers;
4758
cards: typeof cards;
4859
dealCards: typeof dealCards;
4960
functions: typeof functions;
5061
games: typeof games;
62+
http: typeof http;
5163
"lib/functions": typeof lib_functions;
5264
"lib/middlewareUtils": typeof lib_middlewareUtils;
5365
"lib/validators": typeof lib_validators;

‎convex/analyze/group0.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
2+
3+
import { internalAction } from "../_generated/server.js";
4+
import { analyzeModuleGroups } from "./helpers";
5+
import * as players from "../players.js";
6+
import * as prosetHelpers from "../prosetHelpers.js";
7+
import * as queries_getOngoingGames from "../queries/getOngoingGames.js";
8+
import * as revealProset from "../revealProset.js";
9+
import * as types_game_info from "../types/game_info.js";
10+
export default internalAction((ctx) => {
11+
return analyzeModuleGroups({
12+
"players.js": players,
13+
"prosetHelpers.js": prosetHelpers,
14+
"queries/getOngoingGames.js": queries_getOngoingGames,
15+
"revealProset.js": revealProset,
16+
"types/game_info.js": types_game_info,
17+
})
18+
})
19+

‎convex/analyze/group1.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
2+
3+
import { internalAction } from "../_generated/server.js";
4+
import { analyzeModuleGroups } from "./helpers";
5+
import * as types_player_colors from "../types/player_colors.js";
6+
import * as users from "../users.js";
7+
import * as lib_middlewareUtils from "../lib/middlewareUtils.js";
8+
import * as lib_validators from "../lib/validators.js";
9+
import * as message from "../message.js";
10+
export default internalAction((ctx) => {
11+
return analyzeModuleGroups({
12+
"types/player_colors.js": types_player_colors,
13+
"users.js": users,
14+
"lib/middlewareUtils.js": lib_middlewareUtils,
15+
"lib/validators.js": lib_validators,
16+
"message.js": message,
17+
})
18+
})
19+

0 commit comments

Comments
 (0)