Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: make deploy zip size and zip file count available #1403

Merged
merged 9 commits into from
Aug 26, 2024
2,081 changes: 438 additions & 1,643 deletions CHANGELOG.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions src/client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export {
PackageOptions,
RetrieveOptions,
DeployVersionData,
DeployZipData,
RetrieveVersionData,
MetadataApiRetrieveOptions,
} from './types';
53 changes: 40 additions & 13 deletions src/client/metadataApiDeploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ export class DeployResult implements MetadataTransferResult {
public constructor(
public readonly response: MetadataApiDeployStatus,
public readonly components?: ComponentSet,
public readonly replacements = new Map<string, string[]>()
public readonly replacements = new Map<string, string[]>(),
public readonly zipMeta?: { zipSize: number; zipFileCount?: number }
) {}

public getFileResponses(): FileResponse[] {
Expand Down Expand Up @@ -97,6 +98,8 @@ export class MetadataApiDeploy extends MetadataTransfer<
// from the apiOptions and we need it for telemetry.
private readonly isRestDeploy: boolean;
private readonly registry: RegistryAccess;
private zipSize?: number;
private zipFileCount?: number;

public constructor(options: MetadataApiDeployOptions) {
super(options);
Expand Down Expand Up @@ -207,7 +210,10 @@ export class MetadataApiDeploy extends MetadataTransfer<
})
);

const [zipBuffer] = await Promise.all([this.getZipBuffer(), this.maybeSaveTempDirectory('metadata')]);
const [{ zipBuffer, zipFileCount }] = await Promise.all([
this.getZipBuffer(),
this.maybeSaveTempDirectory('metadata'),
]);
// SDR modifies what the mdapi expects by adding a rest param
const { rest, ...optionsWithoutRest } = this.options.apiOptions ?? {};

Expand All @@ -217,7 +223,17 @@ export class MetadataApiDeploy extends MetadataTransfer<
const manifestMsg = manifestVersion ? ` in v${manifestVersion} shape` : '';
const debugMsg = format(`Deploying metadata source%s using ${webService} v${apiVersion}`, manifestMsg);
this.logger.debug(debugMsg);

// Event and Debug output for the zip file used for deploy
this.zipSize = zipBuffer.byteLength;
let zipMessage = `Deployment zip file size = ${this.zipSize} Bytes`;
if (zipFileCount) {
this.zipFileCount = zipFileCount;
zipMessage += ` containing ${zipFileCount} files`;
}
this.logger.debug(zipMessage);
await LifecycleInstance.emit('apiVersionDeploy', { webService, manifestVersion, apiVersion });
await LifecycleInstance.emit('deployZipData', { zipSize: this.zipSize, zipFileCount });

return this.isRestDeploy
? connection.metadata.deployRest(zipBuffer, optionsWithoutRest)
Expand Down Expand Up @@ -266,6 +282,8 @@ export class MetadataApiDeploy extends MetadataTransfer<
numberTestsTotal: result.numberTestsTotal,
testsTotalTime: result.details?.runTestResult?.totalTime,
filesWithReplacementsQuantity: this.replacements.size ?? 0,
zipSize: this.zipSize ?? 0,
zipFileCount: this.zipFileCount ?? 0,
});
} catch (err) {
const error = err as Error;
Expand All @@ -278,7 +296,8 @@ export class MetadataApiDeploy extends MetadataTransfer<
const deployResult = new DeployResult(
result,
this.components,
new Map(Array.from(this.replacements).map(([k, v]) => [k, Array.from(v)]))
new Map(Array.from(this.replacements).map(([k, v]) => [k, Array.from(v)])),
{ zipSize: this.zipSize ?? 0, zipFileCount: this.zipFileCount }
);
// only do event hooks if source, (NOT a metadata format) deploy
if (this.options.components) {
Expand All @@ -292,14 +311,17 @@ export class MetadataApiDeploy extends MetadataTransfer<
return deployResult;
}

private async getZipBuffer(): Promise<Buffer> {
private async getZipBuffer(): Promise<{ zipBuffer: Buffer; zipFileCount?: number }> {
const mdapiPath = this.options.mdapiPath;

// Zip a directory of metadata format source
if (mdapiPath) {
if (!fs.existsSync(mdapiPath) || !fs.lstatSync(mdapiPath).isDirectory()) {
throw messages.createError('error_directory_not_found_or_not_directory', [mdapiPath]);
}

const zip = JSZip();
let zipFileCount = 0;

const zipDirRecursive = (dir: string): void => {
const dirents = fs.readdirSync(dir, { withFileTypes: true });
Expand All @@ -313,33 +335,38 @@ export class MetadataApiDeploy extends MetadataTransfer<
// Ensure only posix paths are added to zip files
const relPosixPath = relPath.replace(/\\/g, '/');
zip.file(relPosixPath, fs.createReadStream(fullPath));
zipFileCount++;
}
}
};
this.logger.debug('Zipping directory for metadata deploy:', mdapiPath);
zipDirRecursive(mdapiPath);

return zip.generateAsync({
type: 'nodebuffer',
compression: 'DEFLATE',
compressionOptions: { level: 9 },
});
return {
zipBuffer: await zip.generateAsync({
type: 'nodebuffer',
compression: 'DEFLATE',
compressionOptions: { level: 9 },
}),
zipFileCount,
};
}
// read the zip into a buffer
// Read a zip of metadata format source into a buffer
if (this.options.zipPath) {
if (!fs.existsSync(this.options.zipPath)) {
throw new SfError(messages.getMessage('error_path_not_found', [this.options.zipPath]));
}
// does encoding matter for zip files? I don't know
return fs.promises.readFile(this.options.zipPath);
return { zipBuffer: await fs.promises.readFile(this.options.zipPath) };
}
// Convert a ComponentSet of metadata in source format and zip
if (this.options.components && this.components) {
const converter = new MetadataConverter(this.registry);
const { zipBuffer } = await converter.convert(this.components, 'metadata', { type: 'zip' });
const { zipBuffer, zipFileCount } = await converter.convert(this.components, 'metadata', { type: 'zip' });
if (!zipBuffer) {
throw new SfError(messages.getMessage('zipBufferError'));
}
return zipBuffer;
return { zipBuffer, zipFileCount };
}
throw new Error('Options should include components, zipPath, or mdapiPath');
}
Expand Down
8 changes: 8 additions & 0 deletions src/client/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,14 @@ export type DeployVersionData = {
webService: 'SOAP' | 'REST';
};

/**
* Data about a deployment zip file being sent to the Metadata API.
*/
export type DeployZipData = {
zipSize: number;
zipFileCount: number;
};

export type RetrieveVersionData = {
apiVersion: string;
manifestVersion: string;
Expand Down
2 changes: 1 addition & 1 deletion src/convert/metadataConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ const getResult =
if ('addToZip' in writer) {
const buffer = writer.buffer;
if (!packagePath) {
return { packagePath, zipBuffer: buffer };
return { packagePath, zipBuffer: buffer, zipFileCount: writer.fileCount };
} else if (buffer) {
await promises.writeFile(packagePath, buffer);
return { packagePath };
Expand Down
5 changes: 5 additions & 0 deletions src/convert/streams.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ export class StandardWriter extends ComponentWriter {
}

export class ZipWriter extends ComponentWriter {
/**
* Count of files (not directories) added to the zip file.
*/
public fileCount: number = 0;
private zip = JSZip();
private zipBuffer?: Buffer;

Expand Down Expand Up @@ -244,6 +248,7 @@ export class ZipWriter extends ComponentWriter {
// Ensure only posix paths are added to zip files
const posixPath = path.replace(/\\/g, '/');
this.zip.file(posixPath, contents);
this.fileCount++;
}
}

Expand Down
4 changes: 4 additions & 0 deletions src/convert/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ export type ConvertResult = {
* Buffer of converted package. `Undefined` if `outputDirectory` is omitted from zip output config.
*/
zipBuffer?: Buffer;
/**
* When a zip buffer is created, this is the number of files in the zip.
*/
zipFileCount?: number;
/**
* Converted source components. Not set if archiving the package.
*/
Expand Down
1 change: 1 addition & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ export {
PackageOptions,
RetrieveOptions,
DeployVersionData,
DeployZipData,
RetrieveVersionData,
} from './client';
export {
Expand Down
3 changes: 2 additions & 1 deletion test/client/metadataApiDeploy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ describe('MetadataApiDeploy', () => {

await operation.start();
const result = await operation.pollStatus();
const expected = new DeployResult(response, deployedComponents);
const zipMeta = { zipSize: 4, zipFileCount: undefined };
const expected = new DeployResult(response, deployedComponents, undefined, zipMeta);

expect(result).to.deep.equal(expected);
});
Expand Down
1 change: 1 addition & 0 deletions test/convert/metadataConverter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ describe('MetadataConverter', () => {
const result = await converter.convert(components, 'metadata', { type: 'zip' });

expect(result.zipBuffer).to.deep.equal(testBuffer);
expect(result.zipFileCount).to.equal(1);
});

it('should return packagePath in result', async () => {
Expand Down
2 changes: 2 additions & 0 deletions test/convert/streams.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,7 @@ describe('Streams', () => {
// NOTE: Zips must only contain files with posix paths
expect(jsZipFileStub.firstCall.args[0]).to.equal('classes/myComponent.cls-meta.xml');
expect(jsZipFileStub.firstCall.args[1]).to.deep.equal(Buffer.from('hi'));
expect(writer.fileCount).to.equal(3);
});

it('should add entries to zip based on given write infos when zip is in-memory only', async () => {
Expand All @@ -495,6 +496,7 @@ describe('Streams', () => {
});
expect(jsZipFileStub.firstCall.args[0]).to.equal('classes/myComponent.cls-meta.xml');
expect(jsZipFileStub.firstCall.args[1]).to.deep.equal(Buffer.from('hi'));
expect(writer.fileCount).to.equal(3);
});

it('should generateAsync zip when stream is finished', async () => {
Expand Down
Loading