Skip to content

Commit

Permalink
feat: Partial execution of pipelines (#821)
Browse files Browse the repository at this point in the history 
- pipelines can now be partially executed by providing the target
placeholder to calculate
- `executePipeline` now accepts an optional target placeholder name
- added a test for partial execution and all possible code dependencies
- fixed block lambdas not generating when used inside of an assignment

---------

Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com>
  • Loading branch information
@WinPlay02 @megalinter-bot
WinPlay02 and megalinter-bot authored Jan 10, 2024
1 parent 7c2526d commit 1e0d03b
Show file tree
Hide file tree
Showing 30 changed files with 558 additions and 15 deletions.
1 change: 1 addition & 0 deletions packages/safe-ds-cli/src/cli/generate.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export const generate = async (fsPaths: string[], options: GenerateOptions): Pro
const generatedFiles = services.generation.PythonGenerator.generate(document, {
destination: URI.file(path.resolve(options.out)),
createSourceMaps: options.sourcemaps,
targetPlaceholder: undefined,
});

for (const file of generatedFiles) {
Expand Down
110 changes: 98 additions & 12 deletions packages/safe-ds-lang/src/language/generation/safe-ds-python-generator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import {
SdsParameter,
SdsParameterList,
SdsPipeline,
SdsPlaceholder,
SdsSegment,
SdsStatement,
} from '../generated/ast.js';
Expand All @@ -75,6 +76,7 @@ import {
getImportedDeclarations,
getImports,
getModuleMembers,
getPlaceholderByName,
getStatements,
Parameter,
streamBlockLambdaResults,
Expand All @@ -90,6 +92,7 @@ import {
import { SafeDsPartialEvaluator } from '../partialEvaluation/safe-ds-partial-evaluator.js';
import { SafeDsServices } from '../safe-ds-module.js';
import { SafeDsPurityComputer } from '../purity/safe-ds-purity-computer.js';
import { ImpurityReason } from '../purity/model.js';

export const CODEGEN_PREFIX = '__gen_';
const BLOCK_LAMBDA_PREFIX = `${CODEGEN_PREFIX}block_lambda_`;
Expand Down Expand Up @@ -129,7 +132,7 @@ export class SafeDsPythonGenerator {
const parentDirectoryPath = path.join(generateOptions.destination!.fsPath, ...packagePath);

const generatedFiles = new Map<string, string>();
const generatedModule = this.generateModule(node);
const generatedModule = this.generateModule(node, generateOptions.targetPlaceholder);
const { text, trace } = toStringAndTrace(generatedModule);
const pythonOutputPath = `${path.join(parentDirectoryPath, this.formatGeneratedFileName(name))}.py`;
if (generateOptions.createSourceMaps) {
Expand Down Expand Up @@ -231,14 +234,14 @@ export class SafeDsPythonGenerator {
return moduleName.replaceAll('%2520', '_').replaceAll(/[ .-]/gu, '_').replaceAll(/\\W/gu, '');
}

private generateModule(module: SdsModule): CompositeGeneratorNode {
private generateModule(module: SdsModule, targetPlaceholder: string | undefined): CompositeGeneratorNode {
const importSet = new Map<String, ImportData>();
const segments = getModuleMembers(module)
.filter(isSdsSegment)
.map((segment) => this.generateSegment(segment, importSet));
const pipelines = getModuleMembers(module)
.filter(isSdsPipeline)
.map((pipeline) => this.generatePipeline(pipeline, importSet));
.map((pipeline) => this.generatePipeline(pipeline, importSet, targetPlaceholder));
const imports = this.generateImports(Array.from(importSet.values()));
const output = new CompositeGeneratorNode();
output.trace(module);
Expand Down Expand Up @@ -271,7 +274,7 @@ export class SafeDsPythonGenerator {
private generateSegment(segment: SdsSegment, importSet: Map<String, ImportData>): CompositeGeneratorNode {
const infoFrame = new GenerationInfoFrame(importSet);
const segmentResult = segment.resultList?.results || [];
let segmentBlock = this.generateBlock(segment.body, infoFrame);
const segmentBlock = this.generateBlock(segment.body, infoFrame);
if (segmentResult.length !== 0) {
segmentBlock.appendNewLine();
segmentBlock.append(
Expand Down Expand Up @@ -317,8 +320,12 @@ export class SafeDsPythonGenerator {
}`;
}

private generatePipeline(pipeline: SdsPipeline, importSet: Map<String, ImportData>): CompositeGeneratorNode {
const infoFrame = new GenerationInfoFrame(importSet, true);
private generatePipeline(
pipeline: SdsPipeline,
importSet: Map<String, ImportData>,
targetPlaceholder: string | undefined,
): CompositeGeneratorNode {
const infoFrame = new GenerationInfoFrame(importSet, true, targetPlaceholder);
return expandTracedToNode(pipeline)`def ${traceToNode(
pipeline,
'name',
Expand Down Expand Up @@ -368,7 +375,11 @@ export class SafeDsPythonGenerator {
frame: GenerationInfoFrame,
generateLambda: boolean = false,
): CompositeGeneratorNode {
const targetPlaceholder = getPlaceholderByName(block, frame.targetPlaceholder);
let statements = getStatements(block).filter((stmt) => this.purityComputer.statementDoesSomething(stmt));
if (targetPlaceholder) {
statements = this.getStatementsNeededForPartialExecution(targetPlaceholder, statements);
}
if (statements.length === 0) {
return traceToNode(block)('pass');
}
Expand All @@ -381,20 +392,88 @@ export class SafeDsPythonGenerator {
)!;
}

private getStatementsNeededForPartialExecution(
targetPlaceholder: SdsPlaceholder,
statementsWithEffect: SdsStatement[],
): SdsStatement[] {
// Find assignment of placeholder, to search used placeholders and impure dependencies
const assignment = getContainerOfType(targetPlaceholder, isSdsAssignment);
if (!assignment || !assignment.expression) {
/* c8 ignore next 2 */
throw new Error(`No assignment for placeholder: ${targetPlaceholder.name}`);
}
// All collected referenced placeholders that are needed for calculating the target placeholder. An expression in the assignment will always exist here
const referencedPlaceholders = new Set<SdsPlaceholder>(
streamAllContents(assignment.expression!)
.filter(isSdsReference)
.filter((reference) => isSdsPlaceholder(reference.target.ref))
.map((reference) => <SdsPlaceholder>reference.target.ref!)
.toArray(),
);
const impurityReasons = new Set<ImpurityReason>(this.purityComputer.getImpurityReasonsForStatement(assignment));
const collectedStatements: SdsStatement[] = [assignment];
for (const prevStatement of statementsWithEffect.reverse()) {
// Statements after the target assignment can always be skipped
if (prevStatement.$containerIndex! >= assignment.$containerIndex!) {
continue;
}
const prevStmtImpurityReasons: ImpurityReason[] =
this.purityComputer.getImpurityReasonsForStatement(prevStatement);
if (
// Placeholder is relevant
(isSdsAssignment(prevStatement) &&
getAssignees(prevStatement)
.filter(isSdsPlaceholder)
.some((prevPlaceholder) => referencedPlaceholders.has(prevPlaceholder))) ||
// Impurity is relevant
prevStmtImpurityReasons.some((pastReason) =>
Array.from(impurityReasons).some((futureReason) =>
pastReason.canAffectFutureImpurityReason(futureReason),
),
)
) {
collectedStatements.push(prevStatement);
// Collect all referenced placeholders
if (isSdsExpressionStatement(prevStatement) || isSdsAssignment(prevStatement)) {
streamAllContents(prevStatement.expression!)
.filter(isSdsReference)
.filter((reference) => isSdsPlaceholder(reference.target.ref))
.map((reference) => <SdsPlaceholder>reference.target.ref!)
.forEach((prevPlaceholder) => {
referencedPlaceholders.add(prevPlaceholder);
});
}
// Collect impurity reasons
prevStmtImpurityReasons.forEach((prevReason) => {
impurityReasons.add(prevReason);
});
}
}
// Get all statements in sorted order
return collectedStatements.reverse();
}

private generateStatement(
statement: SdsStatement,
frame: GenerationInfoFrame,
generateLambda: boolean,
): CompositeGeneratorNode {
const blockLambdaCode: CompositeGeneratorNode[] = [];
if (isSdsAssignment(statement)) {
return traceToNode(statement)(this.generateAssignment(statement, frame, generateLambda));
if (statement.expression) {
for (const lambda of streamAllContents(statement.expression).filter(isSdsBlockLambda)) {
blockLambdaCode.push(this.generateBlockLambda(lambda, frame));
}
}
blockLambdaCode.push(this.generateAssignment(statement, frame, generateLambda));
return joinTracedToNode(statement)(blockLambdaCode, (stmt) => stmt, {
separator: NL,
})!;
} else if (isSdsExpressionStatement(statement)) {
const blockLambdaCode: CompositeGeneratorNode[] = [];
for (const lambda of streamAllContents(statement.expression).filter(isSdsBlockLambda)) {
blockLambdaCode.push(this.generateBlockLambda(lambda, frame));
}
blockLambdaCode.push(this.generateExpression(statement.expression, frame));

return joinTracedToNode(statement)(blockLambdaCode, (stmt) => stmt, {
separator: NL,
})!;
Expand All @@ -415,7 +494,7 @@ export class SafeDsPythonGenerator {
const assignees = getAssignees(assignment);
if (assignees.some((value) => !isSdsWildcard(value))) {
const actualAssignees = assignees.map(this.generateAssignee);
let assignmentStatements = [];
const assignmentStatements = [];
if (requiredAssignees === actualAssignees.length) {
assignmentStatements.push(
expandTracedToNode(assignment)`${joinToNode(actualAssignees, (actualAssignee) => actualAssignee, {
Expand Down Expand Up @@ -473,7 +552,7 @@ export class SafeDsPythonGenerator {

private generateBlockLambda(blockLambda: SdsBlockLambda, frame: GenerationInfoFrame): CompositeGeneratorNode {
const results = streamBlockLambdaResults(blockLambda).toArray();
let lambdaBlock = this.generateBlock(blockLambda.body, frame, true);
const lambdaBlock = this.generateBlock(blockLambda.body, frame, true);
if (results.length !== 0) {
lambdaBlock.appendNewLine();
lambdaBlock.append(
Expand Down Expand Up @@ -812,11 +891,17 @@ class GenerationInfoFrame {
private readonly blockLambdaManager: IdManager<SdsBlockLambda>;
private readonly importSet: Map<String, ImportData>;
public readonly isInsidePipeline: boolean;
public readonly targetPlaceholder: string | undefined;

constructor(importSet: Map<String, ImportData> = new Map<String, ImportData>(), insidePipeline: boolean = false) {
constructor(
importSet: Map<String, ImportData> = new Map<String, ImportData>(),
insidePipeline: boolean = false,
targetPlaceholder: string | undefined = undefined,
) {
this.blockLambdaManager = new IdManager<SdsBlockLambda>();
this.importSet = importSet;
this.isInsidePipeline = insidePipeline;
this.targetPlaceholder = targetPlaceholder;
}

addImport(importData: ImportData | undefined) {
Expand All @@ -836,4 +921,5 @@ class GenerationInfoFrame {
export interface GenerateOptions {
destination: URI;
createSourceMaps: boolean;
targetPlaceholder: string | undefined;
}
4 changes: 4 additions & 0 deletions packages/safe-ds-lang/src/language/helpers/nodeProperties.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,10 @@ export const streamPlaceholders = (node: SdsBlock | undefined): Stream<SdsPlaceh
return stream(getStatements(node)).filter(isSdsAssignment).flatMap(getAssignees).filter(isSdsPlaceholder);
};

export const getPlaceholderByName = (block: SdsBlock, name: string | undefined): SdsPlaceholder | undefined => {
return name ? streamPlaceholders(block).find((placeholder) => placeholder.name === name) : undefined;
};

export const getResults = (node: SdsResultList | undefined): SdsResult[] => {
return node?.results ?? [];
};
Expand Down
40 changes: 40 additions & 0 deletions packages/safe-ds-lang/src/language/purity/model.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ export abstract class ImpurityReason {
* Returns a string representation of this impurity reason.
*/
abstract toString(): string;

/**
* Returns whether this impurity reason can affect a future impurity reason.
* @param future Future Impurity reason to test, if this reason may have an effect on it.
*/
abstract canAffectFutureImpurityReason(future: ImpurityReason): boolean;
}

/**
Expand Down Expand Up @@ -46,6 +52,11 @@ export class FileRead extends ImpurityReason {
return 'File read from ?';
}
}

override canAffectFutureImpurityReason(_future: ImpurityReason): boolean {
// Reads can't affect other reasons
return false;
}
}

/**
Expand Down Expand Up @@ -73,6 +84,16 @@ export class FileWrite extends ImpurityReason {
return 'File write to ?';
}
}

override canAffectFutureImpurityReason(future: ImpurityReason): boolean {
if (future instanceof FileWrite || future instanceof FileRead) {
if (typeof this.path === 'string' && typeof future.path === 'string') {
// Writes only have an effect on other reads and writes, if the files is known and is the same
return this.path === future.path;
}
}
return future !== EndlessRecursion;
}
}

/**
Expand All @@ -98,6 +119,10 @@ export class PotentiallyImpureParameterCall extends ImpurityReason {
return 'Potentially impure call of ?';
}
}

override canAffectFutureImpurityReason(future: ImpurityReason): boolean {
return future !== EndlessRecursion;
}
}

/**
Expand All @@ -113,6 +138,11 @@ class UnknownCallableCallClass extends ImpurityReason {
override toString(): string {
return 'Unknown callable call';
}

canAffectFutureImpurityReason(future: ImpurityReason): boolean {
/* c8 ignore next 2 */
return future !== EndlessRecursion;
}
}

export const UnknownCallableCall = new UnknownCallableCallClass();
Expand All @@ -130,6 +160,12 @@ class EndlessRecursionClass extends ImpurityReason {
override toString(): string {
return 'Endless recursion';
}

override canAffectFutureImpurityReason(_future: ImpurityReason): boolean {
/* c8 ignore next 3 */
// Endless recursions don't have any effect on others
return false;
}
}

export const EndlessRecursion = new EndlessRecursionClass();
Expand All @@ -147,6 +183,10 @@ class OtherImpurityReasonClass extends ImpurityReason {
override toString(): string {
return 'Other';
}

canAffectFutureImpurityReason(future: ImpurityReason): boolean {
return future !== EndlessRecursion;
}
}

/**
Expand Down
21 changes: 21 additions & 0 deletions packages/safe-ds-lang/src/language/purity/safe-ds-purity-computer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,27 @@ export class SafeDsPurityComputer {
return this.getImpurityReasons(node, substitutions);
}

/**
* Returns the reasons why the given statement is impure.
*
* @param node
* The statement to check.
*
* @param substitutions
* The parameter substitutions to use. These are **not** the argument of a call, but the values of the parameters
* of any containing callables, i.e. the context of the node.
*/
getImpurityReasonsForStatement(node: SdsStatement | undefined, substitutions = NO_SUBSTITUTIONS): ImpurityReason[] {
if (isSdsAssignment(node)) {
return this.getImpurityReasonsForExpression(node.expression, substitutions);
} else if (isSdsExpressionStatement(node)) {
return this.getImpurityReasonsForExpression(node.expression, substitutions);
} else {
/* c8 ignore next 2 */
return [];
}
}

/**
* Returns the reasons why the given expression is impure.
*
Expand Down
13 changes: 12 additions & 1 deletion packages/safe-ds-lang/tests/language/generation/safe-ds-python-generator.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest';
import { NodeFileSystem } from 'langium/node';
import { createGenerationTests } from './creator.js';
import { loadDocuments } from '../../helpers/testResources.js';
import { stream } from 'langium';
import { stream, URI } from 'langium';

const services = (await createSafeDsServicesWithBuiltins(NodeFileSystem)).SafeDs;
const pythonGenerator = services.generation.PythonGenerator;
Expand All @@ -19,12 +19,23 @@ describe('generation', async () => {
// Load all documents
const documents = await loadDocuments(services, test.inputUris);

// Get target placeholder name for "run until"
let runUntilPlaceholderName: string | undefined = undefined;

if (test.runUntil) {
const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(
URI.parse(test.runUntil.uri),
);
runUntilPlaceholderName = document.textDocument.getText(test.runUntil.range);
}

// Generate code for all documents
const actualOutputs = stream(documents)
.flatMap((document) =>
pythonGenerator.generate(document, {
destination: test.actualOutputRoot,
createSourceMaps: true,
targetPlaceholder: runUntilPlaceholderName,
}),
)
.map((textDocument) => [textDocument.uri, textDocument.getText()])
Expand Down
Loading

0 comments on commit 1e0d03b

Please sign in to comment.