Skip to content
This repository was archived by the owner on Feb 6, 2026. It is now read-only.

Commit 2509188

Browse files
minherzminherz
committed
feat: auto-populate metadata of log entries at write() (#803)
Populate empty metadata fields of each log entry on write().
1 parent b7612fb commit 2509188

File tree

7 files changed

+277
-5
lines changed

7 files changed

+277
-5
lines changed

‎google-cloud-logging/src/main/java/com/google/cloud/logging/Context.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public static final class Builder {
4949

5050
/** Sets the HTTP request. */
5151
public Builder setRequest(HttpRequest request) {
52-
this.requestBuilder = request.toBuilder();
52+
this.requestBuilder = request != null ? request.toBuilder() : HttpRequest.newBuilder();
5353
return this;
5454
}
5555

‎google-cloud-logging/src/main/java/com/google/cloud/logging/LoggingImpl.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static com.google.cloud.logging.Logging.WriteOption.OptionType.LOG_NAME;
2626
import static com.google.cloud.logging.Logging.WriteOption.OptionType.RESOURCE;
2727

28+
import com.google.api.client.util.Strings;
2829
import com.google.api.core.ApiFunction;
2930
import com.google.api.core.ApiFuture;
3031
import com.google.api.core.ApiFutureCallback;
@@ -92,6 +93,7 @@
9293

9394
class LoggingImpl extends BaseService<LoggingOptions> implements Logging {
9495

96+
protected static final String RESOURCE_NAME_FORMAT = "projects/%s/traces/%s";
9597
private static final int FLUSH_WAIT_TIMEOUT_SECONDS = 6;
9698
private final LoggingRpc rpc;
9799
private final Map<Object, ApiFuture<Void>> pendingWrites = new ConcurrentHashMap<>();
@@ -797,6 +799,55 @@ public void write(Iterable<LogEntry> logEntries, WriteOption... options) {
797799
inWriteCall.set(true);
798800

799801
try {
802+
final Map<Option.OptionType, ?> writeOptions = optionMap(options);
803+
final Boolean populateMetadata1 = getOptions().getAutoPopulateMetadata();
804+
final Boolean populateMetadata2 =
805+
WriteOption.OptionType.AUTO_POPULATE_METADATA.get(writeOptions);
806+
807+
if (populateMetadata2 == Boolean.TRUE
808+
|| (populateMetadata2 == null && populateMetadata1 == Boolean.TRUE)) {
809+
final Boolean needDebugInfo =
810+
Iterables.any(
811+
logEntries,
812+
log -> log.getSeverity() == Severity.DEBUG && log.getSourceLocation() == null);
813+
final SourceLocation sourceLocation =
814+
needDebugInfo ? SourceLocation.fromCurrentContext(1) : null;
815+
final MonitoredResource sharedResourceMetadata = RESOURCE.get(writeOptions);
816+
// populate monitored resource metadata by prioritizing the one set via WriteOption
817+
final MonitoredResource resourceMetadata =
818+
sharedResourceMetadata == null
819+
? MonitoredResourceUtil.getResource(getOptions().getProjectId(), null)
820+
: sharedResourceMetadata;
821+
final Context context = (new ContextHandler()).getCurrentContext();
822+
final ArrayList<LogEntry> populatedLogEntries = Lists.newArrayList();
823+
824+
// populate empty metadata fields of log entries before calling write API
825+
for (LogEntry entry : logEntries) {
826+
LogEntry.Builder builder = entry.toBuilder();
827+
if (resourceMetadata != null && entry.getResource() == null) {
828+
builder.setResource(resourceMetadata);
829+
}
830+
if (context != null && entry.getHttpRequest() == null) {
831+
builder.setHttpRequest(context.getHttpRequest());
832+
}
833+
if (context != null && Strings.isNullOrEmpty(entry.getTrace())) {
834+
MonitoredResource resource =
835+
entry.getResource() != null ? entry.getResource() : resourceMetadata;
836+
builder.setTrace(getFormattedTrace(context.getTraceId(), resource));
837+
}
838+
if (context != null && Strings.isNullOrEmpty(entry.getSpanId())) {
839+
builder.setSpanId(context.getSpanId());
840+
}
841+
if (entry.getSeverity() != null
842+
&& entry.getSeverity() == Severity.DEBUG
843+
&& entry.getSourceLocation() == null) {
844+
builder.setSourceLocation(sourceLocation);
845+
}
846+
populatedLogEntries.add(builder.build());
847+
}
848+
logEntries = populatedLogEntries;
849+
}
850+
800851
writeLogEntries(logEntries, options);
801852
if (flushSeverity != null) {
802853
for (LogEntry logEntry : logEntries) {
@@ -824,6 +875,27 @@ public void flush() {
824875
}
825876
}
826877

878+
/**
879+
* Formats trace following resource name template if the resource metadata has project id.
880+
*
881+
* @param traceId A trace id string or {@code null} if trace info is missing.
882+
* @param resource A {@see MonitoredResource} describing environment metadata.
883+
* @return A formatted trace id string.
884+
*/
885+
private String getFormattedTrace(String traceId, MonitoredResource resource) {
886+
if (traceId == null) {
887+
return null;
888+
}
889+
String projectId = null;
890+
if (resource != null) {
891+
projectId = resource.getLabels().getOrDefault(MonitoredResourceUtil.PORJECTID_LABEL, null);
892+
}
893+
if (projectId != null) {
894+
return String.format(RESOURCE_NAME_FORMAT, projectId, traceId);
895+
}
896+
return traceId;
897+
}
898+
827899
/*
828900
* Write logs synchronously or asynchronously based on writeSynchronicity
829901
* setting.

‎google-cloud-logging/src/main/java/com/google/cloud/logging/MonitoredResourceUtil.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
public class MonitoredResourceUtil {
3434

3535
private static final String APPENGINE_LABEL_PREFIX = "appengine.googleapis.com/";
36+
protected static final String PORJECTID_LABEL = Label.ProjectId.getKey();
3637

3738
protected enum Label {
3839
ClusterName("cluster_name"),

‎google-cloud-logging/src/test/java/com/google/cloud/logging/AutoPopulateMetadataTests.java‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.logging;
18+
19+
import static org.easymock.EasyMock.anyObject;
20+
import static org.easymock.EasyMock.capture;
21+
import static org.easymock.EasyMock.createMock;
22+
import static org.easymock.EasyMock.expect;
23+
import static org.easymock.EasyMock.newCapture;
24+
import static org.easymock.EasyMock.replay;
25+
import static org.junit.Assert.assertEquals;
26+
import static org.junit.Assert.assertNull;
27+
28+
import com.google.api.core.ApiFutures;
29+
import com.google.cloud.MonitoredResource;
30+
import com.google.cloud.logging.HttpRequest.RequestMethod;
31+
import com.google.cloud.logging.Logging.WriteOption;
32+
import com.google.cloud.logging.spi.LoggingRpcFactory;
33+
import com.google.cloud.logging.spi.v2.LoggingRpc;
34+
import com.google.common.collect.ImmutableList;
35+
import com.google.logging.v2.WriteLogEntriesRequest;
36+
import com.google.logging.v2.WriteLogEntriesResponse;
37+
import org.easymock.Capture;
38+
import org.junit.After;
39+
import org.junit.Before;
40+
import org.junit.Test;
41+
42+
public class AutoPopulateMetadataTests {
43+
44+
private static final String LOG_NAME = "test-log";
45+
private static final String RESOURCE_PROJECT_ID = "env-project-id";
46+
private static final String LOGGING_PROJECT_ID = "log-project-id";
47+
private static final MonitoredResource RESOURCE =
48+
MonitoredResource.newBuilder("global")
49+
.addLabel(MonitoredResourceUtil.PORJECTID_LABEL, RESOURCE_PROJECT_ID)
50+
.build();
51+
private static final LogEntry SIMPLE_LOG_ENTRY =
52+
LogEntry.newBuilder(Payload.StringPayload.of("hello"))
53+
.setLogName(LOG_NAME)
54+
.setDestination(LogDestinationName.project(LOGGING_PROJECT_ID))
55+
.build();
56+
private static final LogEntry SIMPLE_LOG_ENTRY_WITH_DEBUG =
57+
LogEntry.newBuilder(Payload.StringPayload.of("hello"))
58+
.setLogName(LOG_NAME)
59+
.setSeverity(Severity.DEBUG)
60+
.setDestination(LogDestinationName.project(LOGGING_PROJECT_ID))
61+
.build();
62+
private static final WriteLogEntriesResponse EMPTY_WRITE_RESPONSE =
63+
WriteLogEntriesResponse.newBuilder().build();
64+
private static final HttpRequest HTTP_REQUEST =
65+
HttpRequest.newBuilder()
66+
.setRequestMethod(RequestMethod.GET)
67+
.setRequestUrl("https://example.com")
68+
.setUserAgent("Test User Agent")
69+
.build();
70+
private static final String TRACE_ID = "01010101010101010101010101010101";
71+
private static final String FORMATTED_TRACE_ID =
72+
String.format(LoggingImpl.RESOURCE_NAME_FORMAT, RESOURCE_PROJECT_ID, TRACE_ID);
73+
private static final String SPAN_ID = "1";
74+
75+
private LoggingRpcFactory mockedRpcFactory;
76+
private LoggingRpc mockedRpc;
77+
private Logging logging;
78+
private Capture<WriteLogEntriesRequest> rpcWriteArgument = newCapture();
79+
private ResourceTypeEnvironmentGetter mockedEnvGetter;
80+
81+
@Before
82+
public void setup() {
83+
mockedEnvGetter = createMock(ResourceTypeEnvironmentGetter.class);
84+
mockedRpcFactory = createMock(LoggingRpcFactory.class);
85+
mockedRpc = createMock(LoggingRpc.class);
86+
expect(mockedRpcFactory.create(anyObject(LoggingOptions.class)))
87+
.andReturn(mockedRpc)
88+
.anyTimes();
89+
expect(mockedRpc.write(capture(rpcWriteArgument)))
90+
.andReturn(ApiFutures.immediateFuture(EMPTY_WRITE_RESPONSE));
91+
MonitoredResourceUtil.setEnvironmentGetter(mockedEnvGetter);
92+
// the following mocks generate MonitoredResource instance same as RESOURCE constant
93+
expect(mockedEnvGetter.getAttribute("project/project-id")).andStubReturn(RESOURCE_PROJECT_ID);
94+
expect(mockedEnvGetter.getAttribute("")).andStubReturn(null);
95+
replay(mockedRpcFactory, mockedRpc, mockedEnvGetter);
96+
97+
LoggingOptions options =
98+
LoggingOptions.newBuilder()
99+
.setProjectId(RESOURCE_PROJECT_ID)
100+
.setServiceRpcFactory(mockedRpcFactory)
101+
.build();
102+
logging = options.getService();
103+
}
104+
105+
@After
106+
public void teardown() {
107+
(new ContextHandler()).removeCurrentContext();
108+
}
109+
110+
private void mockCurrentContext(HttpRequest request, String traceId, String spanId) {
111+
Context mockedContext =
112+
Context.newBuilder().setRequest(request).setTraceId(traceId).setSpanId(spanId).build();
113+
(new ContextHandler()).setCurrentContext(mockedContext);
114+
}
115+
116+
@Test
117+
public void testAutoPopulationEnabledInLoggingOptions() {
118+
mockCurrentContext(HTTP_REQUEST, TRACE_ID, SPAN_ID);
119+
120+
logging.write(ImmutableList.of(SIMPLE_LOG_ENTRY));
121+
122+
LogEntry actual = LogEntry.fromPb(rpcWriteArgument.getValue().getEntries(0));
123+
assertEquals(HTTP_REQUEST, actual.getHttpRequest());
124+
assertEquals(FORMATTED_TRACE_ID, actual.getTrace());
125+
assertEquals(SPAN_ID, actual.getSpanId());
126+
assertEquals(RESOURCE, actual.getResource());
127+
}
128+
129+
@Test
130+
public void testAutoPopulationEnabledInWriteOptionsAndDisabledInLoggingOptions() {
131+
// redefine logging option to opt out auto-populating
132+
LoggingOptions options =
133+
logging.getOptions().toBuilder().setAutoPopulateMetadata(false).build();
134+
logging = options.getService();
135+
mockCurrentContext(HTTP_REQUEST, TRACE_ID, SPAN_ID);
136+
137+
logging.write(ImmutableList.of(SIMPLE_LOG_ENTRY), WriteOption.autoPopulateMetadata(true));
138+
139+
LogEntry actual = LogEntry.fromPb(rpcWriteArgument.getValue().getEntries(0));
140+
assertEquals(HTTP_REQUEST, actual.getHttpRequest());
141+
assertEquals(FORMATTED_TRACE_ID, actual.getTrace());
142+
assertEquals(SPAN_ID, actual.getSpanId());
143+
assertEquals(RESOURCE, actual.getResource());
144+
}
145+
146+
@Test
147+
public void testAutoPopulationDisabledInWriteOptions() {
148+
mockCurrentContext(HTTP_REQUEST, TRACE_ID, SPAN_ID);
149+
150+
logging.write(ImmutableList.of(SIMPLE_LOG_ENTRY), WriteOption.autoPopulateMetadata(false));
151+
152+
LogEntry actual = LogEntry.fromPb(rpcWriteArgument.getValue().getEntries(0));
153+
assertNull(actual.getHttpRequest());
154+
assertNull(actual.getTrace());
155+
assertNull(actual.getSpanId());
156+
assertNull(actual.getResource());
157+
}
158+
159+
@Test
160+
public void testSourceLocationPopulation() {
161+
SourceLocation expected = SourceLocation.fromCurrentContext(0);
162+
logging.write(ImmutableList.of(SIMPLE_LOG_ENTRY_WITH_DEBUG));
163+
164+
LogEntry actual = LogEntry.fromPb(rpcWriteArgument.getValue().getEntries(0));
165+
assertEquals(expected.getFile(), actual.getSourceLocation().getFile());
166+
assertEquals(expected.getClass(), actual.getSourceLocation().getClass());
167+
assertEquals(expected.getFunction(), actual.getSourceLocation().getFunction());
168+
assertEquals(new Long(expected.getLine() + 1), actual.getSourceLocation().getLine());
169+
}
170+
171+
@Test
172+
public void testNotFormattedTraceId() {
173+
mockCurrentContext(HTTP_REQUEST, TRACE_ID, SPAN_ID);
174+
175+
final MonitoredResource expectedResource = MonitoredResource.newBuilder("custom").build();
176+
177+
logging.write(ImmutableList.of(SIMPLE_LOG_ENTRY), WriteOption.resource(expectedResource));
178+
179+
LogEntry actual = LogEntry.fromPb(rpcWriteArgument.getValue().getEntries(0));
180+
assertEquals(TRACE_ID, actual.getTrace());
181+
}
182+
183+
@Test
184+
public void testMonitoredResourcePopulationInWriteOptions() {
185+
mockCurrentContext(HTTP_REQUEST, TRACE_ID, SPAN_ID);
186+
187+
final MonitoredResource expectedResource = MonitoredResource.newBuilder("custom").build();
188+
189+
logging.write(ImmutableList.of(SIMPLE_LOG_ENTRY), WriteOption.resource(expectedResource));
190+
191+
LogEntry actual = LogEntry.fromPb(rpcWriteArgument.getValue().getEntries(0));
192+
assertEquals(expectedResource, actual.getResource());
193+
}
194+
}

‎google-cloud-logging/src/test/java/com/google/cloud/logging/BaseSystemTest.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,6 @@ protected static Iterator<LogEntry> waitForLogs(LogName logName) throws Interrup
130130
return waitForLogs(logName, null, 1);
131131
}
132132

133-
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
134-
135133
protected static Iterator<LogEntry> waitForLogs(Logging.EntryListOption[] options, int minLogs)
136134
throws InterruptedException {
137135
Page<LogEntry> page = logging.listLogEntries(options);

‎google-cloud-logging/src/test/java/com/google/cloud/logging/LoggingImplTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,9 @@ public void setUp() {
261261
.setProjectId(PROJECT)
262262
.setServiceRpcFactory(rpcFactoryMock)
263263
.setRetrySettings(ServiceOptions.getNoRetrySettings())
264+
// disable auto-population for LoggingImpl class tests
265+
// see {@see AutoPopulationTests} for auto-population tests
266+
.setAutoPopulateMetadata(false)
264267
.build();
265268

266269
// By default when calling ListLogEntries, we append a filter of last 24 hours.

‎google-cloud-logging/src/test/java/com/google/cloud/logging/LoggingOptionsTest.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
public class LoggingOptionsTest {
2626
private static final Boolean DONT_AUTO_POPULATE_METADATA = false;
27+
private static final String PROJECT_ID = "fake-project-id";
2728

2829
@Test(expected = IllegalArgumentException.class)
2930
public void testNonGrpcTransportOptions() {
@@ -34,13 +35,16 @@ public void testNonGrpcTransportOptions() {
3435
@Test
3536
public void testAutoPopulateMetadataOption() {
3637
LoggingOptions actual =
37-
LoggingOptions.newBuilder().setAutoPopulateMetadata(DONT_AUTO_POPULATE_METADATA).build();
38+
LoggingOptions.newBuilder()
39+
.setProjectId(PROJECT_ID)
40+
.setAutoPopulateMetadata(DONT_AUTO_POPULATE_METADATA)
41+
.build();
3842
assertEquals(DONT_AUTO_POPULATE_METADATA, actual.getAutoPopulateMetadata());
3943
}
4044

4145
@Test
4246
public void testAutoPopulateMetadataDefaultOption() {
43-
LoggingOptions actual = LoggingOptions.getDefaultInstance();
47+
LoggingOptions actual = LoggingOptions.newBuilder().setProjectId(PROJECT_ID).build();
4448
assertEquals(Boolean.TRUE, actual.getAutoPopulateMetadata());
4549
}
4650
}

0 commit comments

Comments
 (0)