Skip to content

Commit

Permalink
test-runner: Handle multiple networks
Browse files Browse the repository at this point in the history
  • Loading branch information
@serban300
serban300 committed Mar 22, 2023
1 parent 63f26bd commit 6a38a6e
Show file tree
Hide file tree
Showing 5 changed files with 143 additions and 72 deletions.
14 changes: 5 additions & 9 deletions javascript/packages/cli/src/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ program
.description("Run tests on the network defined")
.argument("<testFile>", "ZNDSL file (.zndsl) describing the tests")
.argument(
"[runningNetworkSpec]",
"[runningNetworksSpec...]",
"Path to the network spec json, for using a running network for running the test",
)
.action(test);
Expand Down Expand Up @@ -485,11 +485,7 @@ async function spawn(
* @param testFile
* @param runningNetworksSpec
*/
async function test(
testFile: string,
runningNetworkSpec: string | undefined,
_opts: any,
) {
async function test(testFile: string, runningNetworksSpec: string[]) {
const opts = program.opts();

let extension = testFile.slice(testFile.lastIndexOf(".") + 1);
Expand All @@ -512,8 +508,8 @@ async function test(

const configBasePath = path.dirname(testFile);
const env = new Environment(new RelativeLoader([configBasePath]));
const temmplateContent = fs.readFileSync(testFile).toString();
const content = env.renderString(temmplateContent, process.env);
const templateContent = fs.readFileSync(testFile).toString();
const content = env.renderString(templateContent, process.env);

const testName = getTestNameFromFileName(testFile);

Expand All @@ -533,7 +529,7 @@ async function test(
inCI,
opts.spawnConcurrency,
false,
runningNetworkSpec,
runningNetworksSpec,
);
}

Expand Down
31 changes: 30 additions & 1 deletion javascript/packages/orchestrator/src/network.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ export class Network {
const nodes = this.groups[nodeOrGroupName];

if (!nodes)
throw new Error(`Noode or Group: ${nodeOrGroupName} not present`);
throw new Error(`Node or Group: ${nodeOrGroupName} not present`);
return nodes;
}

Expand Down Expand Up @@ -275,6 +275,35 @@ export class Network {
}
}

getNetworkInfo() {
return {
tmpDir: this.tmpDir,
chainSpecPath: this.chainSpecFullPath,
relay: this.relay.map((node: any) => {
const { name, wsUri, prometheusUri, userDefinedTypes } = node;
return { name, wsUri, prometheusUri, userDefinedTypes };
}),
paras: Object.keys(this.paras).reduce((memo: any, paraId: any) => {
const { chainSpecPath, wasmPath, statePath } = this.paras[paraId];
memo[paraId] = { chainSpecPath, wasmPath, statePath };
memo[paraId].nodes = this.paras[paraId].nodes.map((node) => {
return { ...node };
});
return memo;
}, {}),
nodesByName: Object.keys(this.nodesByName).reduce(
(memo: any, nodeName) => {
const { name, wsUri, prometheusUri, userDefinedTypes, parachainId } =
this.nodesByName[nodeName];
memo[nodeName] = { name, wsUri, prometheusUri, userDefinedTypes };
if (parachainId) memo[nodeName].parachainId = parachainId;
return memo;
},
{},
),
};
}

// show links for access and debug
showNetworkInfo(provider: string) {
const logTable = new CreateLogTable({
Expand Down
5 changes: 2 additions & 3 deletions javascript/packages/orchestrator/src/providers/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,12 @@ export abstract class Client {
abstract restartNode(name: string, timeout: number | null): Promise<boolean>;
}

let client: Client;
let client: Client | undefined;
export function getClient(): Client {
if (!client) throw new Error("Client not initialized");
return client;
}

export function setClient(c: Client) {
if (client) throw new Error("Client already initialized");
export function setClient(c: Client | undefined) {
client = c;
}
163 changes: 105 additions & 58 deletions javascript/packages/orchestrator/src/test-runner/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import path from "path";
import { Network, rebuildNetwork } from "../network";
import { start } from "../orchestrator";
import { Providers } from "../providers";
import { setClient } from "../providers/client";
import { LaunchConfig, TestDefinition } from "../types";
import assertions from "./assertions";
import commands from "./commands";
Expand All @@ -27,6 +28,27 @@ export interface BackchannelMap {
[propertyName: string]: any;
}

function findNetwork(
networks: Network[],
nodeOrGroupName?: string,
): Network | undefined {
if (!nodeOrGroupName) {
return networks[0];
}

let network = networks.find((network) => {
try {
network.getNodes(nodeOrGroupName);
return true;
} catch {}
});

if (!network)
throw new Error(`Node or Group: ${nodeOrGroupName} not present`);

return network;
}

async function stopNetwork(
network: Network,
name: string,
Expand Down Expand Up @@ -118,25 +140,32 @@ export async function run(
inCI: boolean = false,
concurrency: number = 1,
silent: boolean = false,
runningNetworkSpecPath: string | undefined,
runningNetworksSpec: string[] | undefined,
) {
setSilent(silent);
let network: Network;
let networks: Network[] = [];
let backchannelMap: BackchannelMap = {};

let suiteName: string = testName;
if (testDef.description) suiteName += `( ${testDef.description} )`;

// read network file
let networkConfigFilePath = fs.existsSync(testDef.network)
? testDef.network
: path.resolve(configBasePath, testDef.network);
const config: LaunchConfig = readNetworkConfig(networkConfigFilePath);
let networkConfigs: LaunchConfig[] = [];
for (let networkConfigFilePath of testDef.networks) {
// read network file
if (!fs.existsSync(networkConfigFilePath))
networkConfigFilePath = path.resolve(
configBasePath,
networkConfigFilePath,
);
let networkConfig = readNetworkConfig(networkConfigFilePath);

// set the provider
if (!networkConfig.settings)
networkConfig.settings = { provider, timeout: DEFAULT_GLOBAL_TIMEOUT };
else networkConfig.settings.provider = provider;

// set the provider
if (!config.settings)
config.settings = { provider, timeout: DEFAULT_GLOBAL_TIMEOUT };
else config.settings.provider = provider;
networkConfigs.push(networkConfig);
}

// find creds file
let credsFile = inCI ? "config" : testDef.creds;
Expand All @@ -158,53 +187,65 @@ export async function run(
if (credsFileExistInPath) creds = credsFileExistInPath + "/" + credsFile;
}

if (!creds && config.settings.provider === "kubernetes")
throw new Error(`Invalid credential file path: ${credsFile}`);
for (let networkConfig of networkConfigs) {
if (!creds && networkConfig.settings.provider === "kubernetes")
throw new Error(`Invalid credential file path: ${credsFile}`);
}

// create suite
const suite = Suite.create(mocha.suite, suiteName);

suite.beforeAll("launching", async function () {
const launchTimeout = config.settings?.timeout || 500;
this.timeout(launchTimeout * 1000);
try {
if (runningNetworkSpecPath)
console.log("runningNetworkSpecPath", runningNetworkSpecPath);
if (!runningNetworkSpecPath) {
console.log(`\n\n\t Launching network... this can take a while.`);
network = await start(creds!, config, {
spawnConcurrency: concurrency,
inCI,
silent,
});
} else {
const runningNetworkSpec: any = require(runningNetworkSpecPath);
if (provider !== runningNetworkSpec.client.providerName)
throw new Error(
`Invalid provider, the provider set doesn't match with the running network definition`,
for (let [networkConfigIdx, networkConfig] of networkConfigs.entries()) {
const launchTimeout = networkConfig.settings?.timeout || 500;
this.timeout(launchTimeout * 1000);

let runningNetworkSpecPath =
runningNetworksSpec && runningNetworksSpec[networkConfigIdx];
try {
if (runningNetworkSpecPath)
console.log("runningNetworkSpecPath", runningNetworkSpecPath);

let network: Network;
if (!runningNetworkSpecPath) {
console.log(
`\n\n\t Launching network ${testDef.networks[networkConfigIdx]} ... this can take a while.`,
);
network = await start(creds!, networkConfig, {
spawnConcurrency: concurrency,
inCI,
silent,
});
} else {
const runningNetworkSpec: any = require(runningNetworkSpecPath);
if (provider !== runningNetworkSpec.client.providerName)
throw new Error(
`Invalid provider, the provider set doesn't match with the running network definition`,
);

const { client, namespace, tmpDir } = runningNetworkSpec;
// initialize the Client
const initClient = Providers.get(
runningNetworkSpec.client.providerName,
).initClient(client.configPath, namespace, tmpDir);
// initialize the network
network = rebuildNetwork(initClient, runningNetworkSpec);
}

network.showNetworkInfo(config.settings.provider);
const { client, namespace, tmpDir } = runningNetworkSpec;
// initialize the Client
const initClient = Providers.get(
runningNetworkSpec.client.providerName,
).initClient(client.configPath, namespace, tmpDir);
// initialize the network
network = rebuildNetwork(initClient, runningNetworkSpec);
}

await sleep(5 * 1000);
return;
} catch (err) {
console.log(
`\n${decorators.red(
"Error launching the network!",
)} \t ${decorators.bright(err)}`,
);
exitMocha(100);
networks.push(network);
network.showNetworkInfo(networkConfig.settings.provider);
} catch (err) {
console.log(
`\n${decorators.red(
"Error launching the network!",
)} \t ${decorators.bright(err)}`,
);
exitMocha(100);
}
}

await sleep(5 * 1000);
return;
});

suite.afterAll("teardown", async function () {
Expand All @@ -218,11 +259,13 @@ export async function run(
);
}

if (network && !network.wasRunning) {
console.log("\n");
const logsPath = await network.dumpLogs(false);
await stopNetwork(network, testDef.network, inCI, failed);
showNetworkLogsLocation(network, logsPath, inCI);
for (let [networkIdx, network] of networks.entries()) {
if (network && !network.wasRunning) {
console.log("\n");
const logsPath = await network.dumpLogs(false);
await stopNetwork(network, testDef.networks[networkIdx], inCI, failed);
showNetworkLogsLocation(network, logsPath, inCI);
}
}
return;
});
Expand All @@ -241,10 +284,14 @@ export async function run(
}

let testFn = generator(assertion.parsed.args);
const test = new Test(
assertion.original_line,
async () => await testFn(network, backchannelMap, configBasePath),
);
const test = new Test(assertion.original_line, async () => {
// Find the first network that contains the node and run the test on it.
let network = findNetwork(networks, assertion.parsed.args.node_name);

setClient(network?.client);
await testFn(network, backchannelMap, configBasePath);
return;
});
suite.addTest(test);
test.timeout(0);
}
Expand Down
2 changes: 1 addition & 1 deletion javascript/packages/orchestrator/src/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ export interface MultiAddressByNode {
}

export interface TestDefinition {
network: string;
networks: string[];
creds: string;
description?: string;
assertions: Assertion[];
Expand Down

0 comments on commit 6a38a6e

Please sign in to comment.