Skip to content

Commit c71d16e

Browse files
ByronByron
committed
fix!: greatly reduce peak memory usage when reading files.
Previously it was possible to cause a memory consumption of 2.5GB by merely reading a specifically tuned configuration file sized a little less than 1MB, a massive 2500x amplification. Data structures have been optimized and premature optimizations removed in favor of much more efficient memory usage. Now the same file causes a 36MB peak memory usage, still quite a lot, but also ~70x less than before.
1 parent 4017e69 commit c71d16e

File tree

10 files changed

+137
-154
lines changed

10 files changed

+137
-154
lines changed

‎gix-config/src/file/init/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ impl<'a> File<'a> {
5353

5454
this.frontmatter_events = frontmatter;
5555

56+
this.sections.reserve(sections.len());
57+
this.section_order.reserve(sections.len());
5658
for section in sections {
5759
this.push_section_internal(crate::file::Section {
5860
header: section.header,
@@ -61,7 +63,6 @@ impl<'a> File<'a> {
6163
id: Default::default(),
6264
});
6365
}
64-
6566
this
6667
}
6768
}

‎gix-config/src/file/mutable/multi_value.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,9 @@ impl<'borrow, 'lookup, 'event> MultiValueMut<'borrow, 'lookup, 'event> {
182182
let key_sep_events = whitespace.key_value_separators();
183183
MultiValueMut::set_offset(offsets, section_id, offset_index, 2 + key_sep_events.len());
184184
section.insert(offset, Event::Value(escape_value(value).into()));
185-
section.insert_many(offset, key_sep_events.into_iter().rev());
185+
section
186+
.splice(offset..offset, key_sep_events.into_iter().rev())
187+
.for_each(|_| {});
186188
section.insert(offset, Event::SectionKey(key.to_owned()));
187189
}
188190

‎gix-config/src/file/mutable/section.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,8 @@ impl<'a, 'event> SectionMut<'a, 'event> {
262262

263263
let sep_events = self.whitespace.key_value_separators();
264264
size += sep_events.len();
265-
body.insert_many(index.0, sep_events.into_iter().rev());
265+
body.splice(index.0..index.0, sep_events.into_iter().rev())
266+
.for_each(|_| {});
266267

267268
body.insert(index.0, Event::SectionKey(key));
268269
size += 1;
@@ -310,7 +311,7 @@ impl<'event> Deref for SectionMut<'_, 'event> {
310311
}
311312

312313
impl<'event> file::section::Body<'event> {
313-
pub(crate) fn as_mut(&mut self) -> &mut parse::section::Events<'event> {
314+
pub(crate) fn as_mut(&mut self) -> &mut Vec<Event<'event>> {
314315
&mut self.0
315316
}
316317
}

‎gix-config/src/file/section/body.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use crate::{
99

1010
/// A opaque type that represents a section body.
1111
#[derive(PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Debug, Default)]
12-
pub struct Body<'event>(pub(crate) crate::parse::section::Events<'event>);
12+
pub struct Body<'event>(pub(crate) Vec<Event<'event>>);
1313

1414
/// Access
1515
impl<'event> Body<'event> {
@@ -163,7 +163,7 @@ impl<'event> Body<'event> {
163163
/// An owning iterator of a section body. Created by [`Body::into_iter`], yielding
164164
/// un-normalized (`key`, `value`) pairs.
165165
// TODO: tests
166-
pub struct BodyIter<'event>(smallvec::IntoIter<[Event<'event>; 64]>);
166+
pub struct BodyIter<'event>(std::vec::IntoIter<Event<'event>>);
167167

168168
impl<'event> IntoIterator for Body<'event> {
169169
type Item = (Key<'event>, Cow<'event, BStr>);

‎gix-config/src/file/tests.rs‎

Lines changed: 44 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -51,19 +51,16 @@ mod try_from {
5151
let mut sections = HashMap::new();
5252
sections.insert(
5353
SectionId(0),
54-
file::section::Body(
55-
vec![
56-
newline_event(),
57-
name_event("a"),
58-
Event::KeyValueSeparator,
59-
value_event("b"),
60-
newline_event(),
61-
name_event("c"),
62-
Event::KeyValueSeparator,
63-
value_event("d"),
64-
]
65-
.into(),
66-
),
54+
file::section::Body(vec![
55+
newline_event(),
56+
name_event("a"),
57+
Event::KeyValueSeparator,
58+
value_event("b"),
59+
newline_event(),
60+
name_event("c"),
61+
Event::KeyValueSeparator,
62+
value_event("d"),
63+
]),
6764
);
6865
sections
6966
};
@@ -96,19 +93,16 @@ mod try_from {
9693
let mut sections = HashMap::new();
9794
sections.insert(
9895
SectionId(0),
99-
file::section::Body(
100-
vec![
101-
newline_event(),
102-
name_event("a"),
103-
Event::KeyValueSeparator,
104-
value_event("b"),
105-
newline_event(),
106-
name_event("c"),
107-
Event::KeyValueSeparator,
108-
value_event("d"),
109-
]
110-
.into(),
111-
),
96+
file::section::Body(vec![
97+
newline_event(),
98+
name_event("a"),
99+
Event::KeyValueSeparator,
100+
value_event("b"),
101+
newline_event(),
102+
name_event("c"),
103+
Event::KeyValueSeparator,
104+
value_event("d"),
105+
]),
112106
);
113107
sections
114108
};
@@ -144,24 +138,21 @@ mod try_from {
144138
let mut sections = HashMap::new();
145139
sections.insert(
146140
SectionId(0),
147-
file::section::Body(
148-
vec![
149-
newline_event(),
150-
name_event("a"),
151-
Event::KeyValueSeparator,
152-
value_event("b"),
153-
newline_event(),
154-
name_event("c"),
155-
Event::KeyValueSeparator,
156-
value_event("d"),
157-
newline_event(),
158-
]
159-
.into(),
160-
),
141+
file::section::Body(vec![
142+
newline_event(),
143+
name_event("a"),
144+
Event::KeyValueSeparator,
145+
value_event("b"),
146+
newline_event(),
147+
name_event("c"),
148+
Event::KeyValueSeparator,
149+
value_event("d"),
150+
newline_event(),
151+
]),
161152
);
162153
sections.insert(
163154
SectionId(1),
164-
file::section::Body(vec![name_event("e"), Event::KeyValueSeparator, value_event("f")].into()),
155+
file::section::Body(vec![name_event("e"), Event::KeyValueSeparator, value_event("f")]),
165156
);
166157
sections
167158
};
@@ -193,24 +184,21 @@ mod try_from {
193184
let mut sections = HashMap::new();
194185
sections.insert(
195186
SectionId(0),
196-
file::section::Body(
197-
vec![
198-
newline_event(),
199-
name_event("a"),
200-
Event::KeyValueSeparator,
201-
value_event("b"),
202-
newline_event(),
203-
name_event("c"),
204-
Event::KeyValueSeparator,
205-
value_event("d"),
206-
newline_event(),
207-
]
208-
.into(),
209-
),
187+
file::section::Body(vec![
188+
newline_event(),
189+
name_event("a"),
190+
Event::KeyValueSeparator,
191+
value_event("b"),
192+
newline_event(),
193+
name_event("c"),
194+
Event::KeyValueSeparator,
195+
value_event("d"),
196+
newline_event(),
197+
]),
210198
);
211199
sections.insert(
212200
SectionId(1),
213-
file::section::Body(vec![name_event("e"), Event::KeyValueSeparator, value_event("f")].into()),
201+
file::section::Body(vec![name_event("e"), Event::KeyValueSeparator, value_event("f")]),
214202
);
215203
sections
216204
};

‎gix-config/src/parse/events.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use smallvec::SmallVec;
44

55
use crate::{
66
parse,
7-
parse::{section, Event, Section},
7+
parse::{Event, Section},
88
};
99

1010
/// A type store without allocation all events that are typically preceding the first section.
@@ -292,7 +292,7 @@ fn from_bytes<'a, 'b>(
292292
filter: Option<fn(&Event<'a>) -> bool>,
293293
) -> Result<Events<'b>, parse::Error> {
294294
let mut header = None;
295-
let mut events = section::Events::default();
295+
let mut events = Vec::with_capacity(256);
296296
let mut frontmatter = FrontMatterEvents::default();
297297
let mut sections = Vec::new();
298298
parse::from_bytes(input, &mut |e: Event<'_>| match e {

‎gix-config/src/parse/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ pub struct Section<'a> {
9393
/// The section name and subsection name, if any.
9494
pub header: section::Header<'a>,
9595
/// The syntactic events found in this section.
96-
pub events: section::Events<'a>,
96+
pub events: Vec<Event<'a>>,
9797
}
9898

9999
/// A parsed comment containing the comment marker and comment.

0 commit comments

Comments
 (0)