Skip to content

Commit 77059e4

Browse files
simschlasimschla
committed
1416: limit output fetched by npm process
usually only the last few lines are of interest anyway, so be kind to memory usage.
1 parent 1c285ec commit 77059e4

File tree

4 files changed

+322
-4
lines changed

4 files changed

+322
-4
lines changed

‎lib/src/main/java/com/diffplug/spotless/LimitedOverwritingByteArrayOutputStream.java

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
* Copyright 2023 DiffPlug
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.diffplug.spotless;
17+
18+
import java.io.ByteArrayOutputStream;
19+
import java.io.IOException;
20+
import java.io.OutputStream;
21+
import java.io.UnsupportedEncodingException;
22+
23+
public class LimitedOverwritingByteArrayOutputStream extends ByteArrayOutputStream {
24+
25+
private final int limit;
26+
27+
private int zeroIndexPointer = 0;
28+
29+
private boolean isOverLimit = false;
30+
31+
public LimitedOverwritingByteArrayOutputStream(int limit) {
32+
this(limit, 32);
33+
}
34+
35+
public LimitedOverwritingByteArrayOutputStream(int limit, int initialCapacity) {
36+
super(initialCapacity);
37+
if (limit < initialCapacity) {
38+
throw new IllegalArgumentException("Limit must be greater than initial capacity");
39+
}
40+
if (limit < 0) {
41+
throw new IllegalArgumentException("Limit must be greater than 0");
42+
}
43+
if (limit % 2 != 0) {
44+
throw new IllegalArgumentException("Limit must be even"); // to fit 16 bit unicode chars
45+
}
46+
this.limit = limit;
47+
}
48+
49+
// ---- writing
50+
@Override
51+
public synchronized void write(int b) {
52+
if (count < limit) {
53+
super.write(b);
54+
return;
55+
}
56+
isOverLimit = true;
57+
buf[zeroIndexPointer] = (byte) b;
58+
zeroIndexPointer = (zeroIndexPointer + 1) % limit;
59+
}
60+
61+
@Override
62+
public synchronized void write(byte[] b, int off, int len) {
63+
int remaining = limit - count;
64+
if (remaining >= len) {
65+
super.write(b, off, len);
66+
return;
67+
}
68+
if (remaining > 0) {
69+
// write what we can "normally"
70+
super.write(b, off, remaining);
71+
// rest delegated
72+
write(b, off + remaining, len - remaining);
73+
return;
74+
}
75+
// we are over the limit
76+
isOverLimit = true;
77+
// write till limit is reached
78+
int writeTillLimit = Math.min(len, limit - zeroIndexPointer);
79+
System.arraycopy(b, off, buf, zeroIndexPointer, writeTillLimit);
80+
zeroIndexPointer = (zeroIndexPointer + writeTillLimit) % limit;
81+
if (writeTillLimit < len) {
82+
// write rest
83+
write(b, off + writeTillLimit, len - writeTillLimit);
84+
}
85+
}
86+
87+
@Override
88+
public synchronized void reset() {
89+
super.reset();
90+
zeroIndexPointer = 0;
91+
isOverLimit = false;
92+
}
93+
94+
// ---- output
95+
@Override
96+
public synchronized void writeTo(OutputStream out) throws IOException {
97+
if (!isOverLimit) {
98+
super.writeTo(out);
99+
return;
100+
}
101+
out.write(buf, zeroIndexPointer, limit - zeroIndexPointer);
102+
out.write(buf, 0, zeroIndexPointer);
103+
}
104+
105+
@Override
106+
public synchronized byte[] toByteArray() {
107+
if (!isOverLimit) {
108+
return super.toByteArray();
109+
}
110+
byte[] result = new byte[limit];
111+
System.arraycopy(buf, zeroIndexPointer, result, 0, limit - zeroIndexPointer);
112+
System.arraycopy(buf, 0, result, limit - zeroIndexPointer, zeroIndexPointer);
113+
return result;
114+
}
115+
116+
@Override
117+
public synchronized String toString() {
118+
if (!isOverLimit) {
119+
return super.toString();
120+
}
121+
return new String(buf, zeroIndexPointer, limit - zeroIndexPointer) + new String(buf, 0, zeroIndexPointer);
122+
}
123+
124+
@Override
125+
public synchronized String toString(String charsetName) throws UnsupportedEncodingException {
126+
if (!isOverLimit) {
127+
return super.toString(charsetName);
128+
}
129+
return new String(buf, zeroIndexPointer, limit - zeroIndexPointer, charsetName) + new String(buf, 0, zeroIndexPointer, charsetName);
130+
}
131+
132+
}

‎lib/src/main/java/com/diffplug/spotless/ProcessRunner.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,17 @@
5252
public class ProcessRunner implements AutoCloseable {
5353
private final ExecutorService threadStdOut = Executors.newSingleThreadExecutor();
5454
private final ExecutorService threadStdErr = Executors.newSingleThreadExecutor();
55-
private final ByteArrayOutputStream bufStdOut = new ByteArrayOutputStream();
56-
private final ByteArrayOutputStream bufStdErr = new ByteArrayOutputStream();
55+
private final ByteArrayOutputStream bufStdOut;
56+
private final ByteArrayOutputStream bufStdErr;
5757

58-
public ProcessRunner() {}
58+
public ProcessRunner() {
59+
this(-1);
60+
}
61+
62+
public ProcessRunner(int limitedBuffers) {
63+
this.bufStdOut = limitedBuffers >= 0 ? new LimitedOverwritingByteArrayOutputStream(limitedBuffers) : new ByteArrayOutputStream();
64+
this.bufStdErr = limitedBuffers >= 0 ? new LimitedOverwritingByteArrayOutputStream(limitedBuffers) : new ByteArrayOutputStream();
65+
}
5966

6067
/** Executes the given shell command (using {@code cmd} on windows and {@code sh} on unix). */
6168
public Result shell(String cmd) throws IOException, InterruptedException {

‎lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class NpmProcess {
4242
this.workingDir = workingDir;
4343
this.npmExecutable = npmExecutable;
4444
this.nodeExecutable = nodeExecutable;
45-
processRunner = new ProcessRunner();
45+
processRunner = new ProcessRunner(100 * 1024); // 100kB
4646
}
4747

4848
void install() {

‎lib/src/test/java/com/diffplug/spotless/LimitedOverwritingByteArrayOutputStreamTest.java

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* Copyright 2023 DiffPlug
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.diffplug.spotless;
17+
18+
import java.io.ByteArrayOutputStream;
19+
import java.io.IOException;
20+
import java.util.stream.Stream;
21+
22+
import org.assertj.core.api.Assertions;
23+
import org.junit.jupiter.api.Test;
24+
import org.junit.jupiter.params.ParameterizedTest;
25+
import org.junit.jupiter.params.provider.Arguments;
26+
import org.junit.jupiter.params.provider.MethodSource;
27+
28+
class LimitedOverwritingByteArrayOutputStreamTest {
29+
30+
private final byte[] bytes = new byte[]{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'};
31+
32+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
33+
@MethodSource("writeStrategies")
34+
void toStringBehavesNormallyWithinLimit(String name, ByteWriteStrategy writeStrategy) {
35+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(13, 1);
36+
writeStrategy.write(stream, bytes);
37+
Assertions.assertThat(stream.toString()).isEqualTo("0123456789");
38+
}
39+
40+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
41+
@MethodSource("writeStrategies")
42+
void toStringBehavesOverwritingOverLimit(String name, ByteWriteStrategy writeStrategy) {
43+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(4, 1);
44+
writeStrategy.write(stream, bytes);
45+
Assertions.assertThat(stream.toString()).hasSize(4);
46+
Assertions.assertThat(stream.toString()).isEqualTo("6789");
47+
}
48+
49+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
50+
@MethodSource("writeStrategies")
51+
void toStringBehavesNormallyAtExactlyLimit(String name, ByteWriteStrategy writeStrategy) {
52+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(bytes.length, 1);
53+
writeStrategy.write(stream, bytes);
54+
Assertions.assertThat(stream.toString()).isEqualTo("0123456789");
55+
}
56+
57+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
58+
@MethodSource("writeStrategies")
59+
void toByteArrayBehavesNormallyWithinLimit(String name, ByteWriteStrategy writeStrategy) {
60+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(13, 1);
61+
writeStrategy.write(stream, bytes);
62+
Assertions.assertThat(stream.toByteArray()).isEqualTo(bytes);
63+
}
64+
65+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
66+
@MethodSource("writeStrategies")
67+
void toByteArrayBehavesOverwritingOverLimit(String name, ByteWriteStrategy writeStrategy) {
68+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(4, 1);
69+
writeStrategy.write(stream, bytes);
70+
Assertions.assertThat(stream.toByteArray()).hasSize(4);
71+
Assertions.assertThat(stream.toByteArray()).isEqualTo(new byte[]{'6', '7', '8', '9'});
72+
}
73+
74+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
75+
@MethodSource("writeStrategies")
76+
void toByteArrayBehavesOverwritingAtExactlyLimit(String name, ByteWriteStrategy writeStrategy) {
77+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(bytes.length, 1);
78+
writeStrategy.write(stream, bytes);
79+
Assertions.assertThat(stream.toByteArray()).isEqualTo(bytes);
80+
}
81+
82+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
83+
@MethodSource("writeStrategies")
84+
void writeToBehavesNormallyWithinLimit(String name, ByteWriteStrategy writeStrategy) throws IOException {
85+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(13, 1);
86+
writeStrategy.write(stream, bytes);
87+
ByteArrayOutputStream target = new ByteArrayOutputStream();
88+
stream.writeTo(target);
89+
Assertions.assertThat(target.toByteArray()).isEqualTo(bytes);
90+
}
91+
92+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
93+
@MethodSource("writeStrategies")
94+
void writeToBehavesOverwritingOverLimit(String name, ByteWriteStrategy writeStrategy) throws IOException {
95+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(4, 1);
96+
writeStrategy.write(stream, bytes);
97+
ByteArrayOutputStream target = new ByteArrayOutputStream();
98+
stream.writeTo(target);
99+
Assertions.assertThat(target.toByteArray()).hasSize(4);
100+
Assertions.assertThat(target.toByteArray()).isEqualTo(new byte[]{'6', '7', '8', '9'});
101+
}
102+
103+
@ParameterizedTest(name = "{index} writeStrategy: {0}")
104+
@MethodSource("writeStrategies")
105+
void writeToBehavesNormallyAtExactlyLimit(String name, ByteWriteStrategy writeStrategy) throws IOException {
106+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(bytes.length, 1);
107+
writeStrategy.write(stream, bytes);
108+
ByteArrayOutputStream target = new ByteArrayOutputStream();
109+
stream.writeTo(target);
110+
Assertions.assertThat(target.toByteArray()).isEqualTo(bytes);
111+
}
112+
113+
@Test
114+
void writeToBehavesCorrectlyWhenOverLimitMultipleCalls() {
115+
// this test explicitly captures a border case where the buffer is not empty but can exactly fit what we are writing
116+
LimitedOverwritingByteArrayOutputStream stream = new LimitedOverwritingByteArrayOutputStream(2, 1);
117+
stream.write('0');
118+
stream.write(new byte[]{'1', '2'}, 0, 2);
119+
Assertions.assertThat(stream.toString()).hasSize(2);
120+
Assertions.assertThat(stream.toString()).isEqualTo("12");
121+
}
122+
123+
private static Stream<Arguments> writeStrategies() {
124+
return Stream.of(
125+
Arguments.of("writeAllAtOnce", allAtOnce()),
126+
Arguments.of("writeOneByteAtATime", oneByteAtATime()),
127+
Arguments.of("writeTwoBytesAtATime", twoBytesAtATime()),
128+
Arguments.of("writeOneAndThenTwoBytesAtATime", oneAndThenTwoBytesAtATime()),
129+
Arguments.of("firstFourBytesAndThenTheRest", firstFourBytesAndThenTheRest()));
130+
}
131+
132+
private static ByteWriteStrategy allAtOnce() {
133+
return (stream, bytes) -> stream.write(bytes, 0, bytes.length);
134+
}
135+
136+
private static ByteWriteStrategy oneByteAtATime() {
137+
return (stream, bytes) -> {
138+
for (byte b : bytes) {
139+
stream.write(b);
140+
}
141+
};
142+
}
143+
144+
private static ByteWriteStrategy twoBytesAtATime() {
145+
return (stream, bytes) -> {
146+
for (int i = 0; i < bytes.length; i += 2) {
147+
stream.write(bytes, i, 2);
148+
}
149+
};
150+
}
151+
152+
private static ByteWriteStrategy oneAndThenTwoBytesAtATime() {
153+
return (stream, bytes) -> {
154+
int written = 0;
155+
for (int i = 0; i + 3 < bytes.length; i += 3) {
156+
stream.write(bytes, i, 1);
157+
stream.write(bytes, i + 1, 2);
158+
written += 3;
159+
}
160+
if (written < bytes.length) {
161+
stream.write(bytes, written, bytes.length - written);
162+
}
163+
164+
};
165+
}
166+
167+
private static ByteWriteStrategy firstFourBytesAndThenTheRest() {
168+
return (stream, bytes) -> {
169+
stream.write(bytes, 0, 4);
170+
stream.write(bytes, 4, bytes.length - 4);
171+
};
172+
}
173+
174+
@FunctionalInterface
175+
private interface ByteWriteStrategy {
176+
void write(LimitedOverwritingByteArrayOutputStream stream, byte[] bytes);
177+
}
178+
179+
}

0 commit comments

Comments
 (0)