fix(vscode): smoother keeps arrow axis perpendicular; dial spacing back

Two adjustments on top of the earlier edge-polish commit, driven by
feedback on the Running composite in ``test_formatter_playground``.

- Arrow perpendicularity. The merge-based smoother collapsed a short
  final stub by deleting its bend point and sliding the preceding
  one, which FLIPPED the final segment's axis — a 2px vertical stub
  into a node's top edge became a long horizontal segment running
  parallel to that edge. The arrow landed parallel instead of
  perpendicular.
  Rewrite the smoother to elongate a stub by shifting the two bend
  points parallel to the stub, keeping the segment's axis intact. An
  anti-cycle guard skips the shift when it would trade a stub at one
  end for a new stub at the other end — the pathological case where
  both ends are already short; in that case the polyline is left as
  ELK produced it so perpendicularity is never sacrificed.

- Edge spacing dialed back. The earlier bump was too aggressive on
  small diagrams. Settle on values that are still noticeably roomier
  than the original:
    elk.spacing.edgeNode                 42 -> 48 (was 56)
    elk.spacing.edgeEdge                 30 -> 36 (was 44)
    elk.layered.spacing.baseValue        48 -> 50 (was 52)
    elk.layered.spacing.edgeNodeBetweenLayers 46 -> 52 (was 58)
    elk.layered.spacing.edgeEdgeBetweenLayers (new) 32 (was 40)
    elk.layered.spacing.nodeNodeBetweenLayers 110 -> 116 (was 120)
  mergeEdges / mergeHierarchyEdges / unnecessaryBendpoints stay on.

The combined effect on the playground's ``[*] -> Active`` edge:
ELK now picks a balanced 10-75-10 orthogonal route instead of the
original 18-75-2, and the smoother leaves it alone because an anti-
cycle shift would cost the other end. The arrow lands vertically on
Active's top edge, well inside the flat region between the rounded
corners.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: HansBug

editors/jsfcstm/src/diagram/elk-graph.ts

@@ -367,18 +367,20 @@ export function buildFcstmElkGraph(
             // Wider breathing room than the first draft — users explicitly
             // asked for a less cramped layout.
             'elk.spacing.nodeNode': '80',
-            'elk.layered.spacing.nodeNodeBetweenLayers': '120',
-            // Push parallel edges further apart so two transitions that
-            // share a layer do not run glued together. Same story for
-            // the edge-vs-node gap — ELK otherwise happily parks a line
-            // one or two pixels off the node outline.
-            'elk.spacing.edgeNode': '56',
-            'elk.spacing.edgeEdge': '44',
+            'elk.layered.spacing.nodeNodeBetweenLayers': '116',
+            // Push parallel edges apart so two transitions that share
+            // a layer do not run glued together, and keep edges a
+            // readable distance from node outlines. The values are a
+            // compromise between the original tight defaults and the
+            // earlier wider bump — larger than before, but not so
+            // aggressive that small diagrams waste whitespace.
+            'elk.spacing.edgeNode': '48',
+            'elk.spacing.edgeEdge': '36',
             'elk.spacing.edgeLabel': '24',
             'elk.spacing.componentComponent': '64',
-            'elk.layered.spacing.baseValue': '52',
-            'elk.layered.spacing.edgeNodeBetweenLayers': '58',
-            'elk.layered.spacing.edgeEdgeBetweenLayers': '40',
+            'elk.layered.spacing.baseValue': '50',
+            'elk.layered.spacing.edgeNodeBetweenLayers': '52',
+            'elk.layered.spacing.edgeEdgeBetweenLayers': '32',
             'elk.edgeLabels.placement': 'CENTER',
             'elk.layered.nodePlacement.favorStraightEdges': 'true',
             // Merging hierarchy edges lets ELK collapse many parallel

editors/vscode/src/preview-webview/render/edge-smoother.ts

@@ -1,16 +1,25 @@
 /**
  * Post-process an ELK-laid-out preview graph so edges do not end (or
- * start) with a tiny orthogonal stub right next to a node. When ELK
- * routes an edge through several bends and the final segment lands
- * within a few pixels of the target port, the visual result is a
- * perpendicular "L" glued to the arrowhead that reads as rendering
- * noise rather than intentional routing.
+ * start) with a tiny orthogonal stub right next to a node. A short
+ * terminal segment reads as visual noise glued to the arrowhead.
  *
- * The repair absorbs a short stub into the neighboring bend instead of
- * shifting both ends — a plain shift can trade one stub for another
- * when a polyline has a short final AND a short initial segment. The
- * pass iterates until the polyline stabilizes, so chains of stubs are
- * collapsed in one go.
+ * The smoother preserves two invariants that the earlier merge-based
+ * version broke:
+ *
+ * 1. Orthogonality — every segment remains axis-aligned.
+ * 2. Axis at the endpoint — the final segment keeps the same axis it
+ *    had before smoothing, so the arrow lands perpendicular to the
+ *    destination node's border (horizontal segment → hits a vertical
+ *    side, vertical segment → hits a horizontal side).
+ *
+ * The fix elongates a short terminal segment by shifting the two bend
+ * points adjacent to it in the direction parallel to the segment,
+ * keeping the penultimate segment perpendicular. If that shift would
+ * turn a formerly-healthy segment on the OTHER side into a new stub,
+ * the fix is skipped — that was the ping-pong bug where end-stub
+ * elongation created a start stub and vice versa. In such a case the
+ * tiny stub is left as-is; preserving perpendicularity matters more
+ * than hiding a two-pixel segment.
  */
 import type {PreviewElkEdge, PreviewElkNode} from '../types';
 
@@ -21,54 +30,100 @@ interface Point {
     y: number;
 }
 
-function segmentLenLInf(a: Point, b: Point): number {
+function manhattan(a: Point, b: Point): number {
     return Math.abs(a.x - b.x) + Math.abs(a.y - b.y);
 }
 
+interface ShiftPlan {
+    axis: 'x' | 'y';
+    /** +1 / -1 — direction along the segment axis that elongates the stub. */
+    sign: number;
+    /** How many pixels to shift. */
+    delta: number;
+    /** Indices of the two points that need to move. */
+    a: number;
+    b: number;
+}
+
 /**
- * Try to collapse the trailing stub of a polyline by removing the
- * useless bend point adjacent to the endpoint. Returns ``true`` if a
- * change was made.
- *
- * Geometrically: when ``p_{n-1}`` is only ``short`` away from ``p_n`` on
- * axis F, and ``p_{n-2}`` → ``p_{n-1}`` runs perpendicular to that (axis
- * P), we can drop ``p_{n-1}`` and slide ``p_{n-2}`` onto ``p_n``'s F
- * coordinate. The previous segment keeps its axis; the new terminal
- * segment runs along P and inherits the length of the old penultimate
- * segment, so no new short stub is introduced as long as the original
- * penultimate segment was long enough.
+ * Build a shift plan for the tail of a polyline. Returns ``null`` when
+ * the tail is not a valid short stub (either already long enough, or
+ * the penultimate geometry is not perpendicular to the final segment).
  */
-function collapseEndStub(points: Point[], threshold: number): boolean {
-    if (points.length < 3) return false;
+function planTailShift(points: Point[], threshold: number): ShiftPlan | null {
+    if (points.length < 3) return null;
     const last = points[points.length - 1];
     const prev = points[points.length - 2];
     const pprev = points[points.length - 3];
     const dx = last.x - prev.x;
     const dy = last.y - prev.y;
     const stub = Math.abs(dx) + Math.abs(dy);
-    if (stub <= 0 || stub >= threshold) return false;
-    const axis: 'x' | 'y' = Math.abs(dx) > Math.abs(dy) ? 'x' : 'y';
-    pprev[axis] = last[axis];
-    points.splice(points.length - 2, 1);
-    return true;
+    if (stub <= 0 || stub >= threshold) return null;
+    const horizontal = Math.abs(dx) >= Math.abs(dy);
+    if (horizontal) {
+        // Penultimate segment must be vertical (same x on both ends).
+        if (Math.abs(pprev.x - prev.x) > 0.5) return null;
+        const sign = dx > 0 ? -1 : 1;
+        return {axis: 'x', sign, delta: threshold - stub, a: points.length - 2, b: points.length - 3};
+    }
+    if (Math.abs(pprev.y - prev.y) > 0.5) return null;
+    const sign = dy > 0 ? -1 : 1;
+    return {axis: 'y', sign, delta: threshold - stub, a: points.length - 2, b: points.length - 3};
 }
 
-function collapseStartStub(points: Point[], threshold: number): boolean {
-    if (points.length < 3) return false;
+function planHeadShift(points: Point[], threshold: number): ShiftPlan | null {
+    if (points.length < 3) return null;
     const first = points[0];
     const next = points[1];
     const nnext = points[2];
     const dx = next.x - first.x;
     const dy = next.y - first.y;
     const stub = Math.abs(dx) + Math.abs(dy);
-    if (stub <= 0 || stub >= threshold) return false;
-    const axis: 'x' | 'y' = Math.abs(dx) > Math.abs(dy) ? 'x' : 'y';
-    nnext[axis] = first[axis];
-    points.splice(1, 1);
-    return true;
+    if (stub <= 0 || stub >= threshold) return null;
+    const horizontal = Math.abs(dx) >= Math.abs(dy);
+    if (horizontal) {
+        if (Math.abs(nnext.x - next.x) > 0.5) return null;
+        const sign = dx > 0 ? 1 : -1;
+        return {axis: 'x', sign, delta: threshold - stub, a: 1, b: 2};
+    }
+    if (Math.abs(nnext.y - next.y) > 0.5) return null;
+    const sign = dy > 0 ? 1 : -1;
+    return {axis: 'y', sign, delta: threshold - stub, a: 1, b: 2};
 }
 
-function smoothEdge(edge: PreviewElkEdge, minStubLen: number): void {
+function applyShift(points: Point[], plan: ShiftPlan): void {
+    const {axis, sign, delta, a, b} = plan;
+    points[a][axis] += sign * delta;
+    points[b][axis] += sign * delta;
+}
+
+/**
+ * Would applying this plan on the polyline create a fresh short stub at
+ * the OPPOSITE end? A plan qualifies as safe only when it leaves both
+ * ends of the polyline above the threshold.
+ */
+function createsOppositeStub(
+    points: Point[],
+    plan: ShiftPlan,
+    threshold: number,
+    end: 'tail' | 'head'
+): boolean {
+    // Clone just enough to check. Applying the shift mutates; test on a
+    // shallow clone of the two points that change.
+    const cloned = points.map(p => ({x: p.x, y: p.y}));
+    applyShift(cloned, plan);
+    if (end === 'tail') {
+        // Check the head segment of the clone.
+        if (cloned.length < 2) return false;
+        const headLen = manhattan(cloned[0], cloned[1]);
+        return headLen > 0 && headLen < threshold;
+    }
+    if (cloned.length < 2) return false;
+    const tailLen = manhattan(cloned[cloned.length - 2], cloned[cloned.length - 1]);
+    return tailLen > 0 && tailLen < threshold;
+}
+
+function smoothEdge(edge: PreviewElkEdge, threshold: number): void {
     for (const section of edge.sections || []) {
         const points: Point[] = [
             {x: section.startPoint.x, y: section.startPoint.y},
@@ -77,14 +132,15 @@ function smoothEdge(edge: PreviewElkEdge, minStubLen: number): void {
         ];
         if (points.length < 3) continue;
 
-        // Iterate until the polyline is stable. Both collapses shrink
-        // the array so the loop is bounded by the original length.
-        let changed = true;
-        let safety = points.length + 2;
-        while (changed && safety-- > 0) {
-            changed = false;
-            if (collapseEndStub(points, minStubLen)) changed = true;
-            if (collapseStartStub(points, minStubLen)) changed = true;
+        // Try tail first; it is the more visually important end
+        // because the arrowhead lives there.
+        const tailPlan = planTailShift(points, threshold);
+        if (tailPlan && !createsOppositeStub(points, tailPlan, threshold, 'tail')) {
+            applyShift(points, tailPlan);
+        }
+        const headPlan = planHeadShift(points, threshold);
+        if (headPlan && !createsOppositeStub(points, headPlan, threshold, 'head')) {
+            applyShift(points, headPlan);
         }
 
         section.startPoint = {x: points[0].x, y: points[0].y};