Skip to content

Commit

Permalink
Bumped avro to 1.11.4 and implemented setter/getter adjustments
Browse files Browse the repository at this point in the history
  • Loading branch information
Dominik Diedrich committed Oct 22, 2024
1 parent 09b3487 commit a0a429c
Show file tree
Hide file tree
Showing 7 changed files with 27 additions and 26 deletions.
2 changes: 1 addition & 1 deletion ...doop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public class TestAvroSerialization {
@Test
public void testSpecific() throws Exception {
AvroRecord before = new AvroRecord();
before.intField = 5;
before.put("intField", 5);
AvroRecord after = SerializationTestUtil.testSerialization(conf, before);
assertEquals(before, after);
}
Expand Down
11 changes: 6 additions & 5 deletions ...client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ public class JobQueueChangeEvent implements HistoryEvent {
private JobQueueChange datum = new JobQueueChange();

public JobQueueChangeEvent(JobID id, String queueName) {
datum.jobid = new Utf8(id.toString());
datum.jobQueueName = new Utf8(queueName);
datum.setJobid(id.toString());
datum.setJobQueueName(queueName);
}

JobQueueChangeEvent() { }
Expand All @@ -54,13 +54,14 @@ public void setDatum(Object datum) {

/** Get the Job ID */
public JobID getJobId() {
return JobID.forName(datum.jobid.toString());
return JobID.forName(datum.getJobid().toString());
}

/** Get the new Job queue name */
public String getJobQueueName() {
if (datum.jobQueueName != null) {
return datum.jobQueueName.toString();
java.lang.CharSequence jobQueueName = datum.getJobQueueName();
if (jobQueueName != null) {
return jobQueueName.toString();
}
return null;
}
Expand Down
2 changes: 1 addition & 1 deletion hadoop-project/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
<java.security.egd>file:///dev/urandom</java.security.egd>

<!-- avro version -->
<avro.version>1.9.2</avro.version>
<avro.version>1.11.4</avro.version>

<!-- jersey version -->
<jersey.version>1.19.4</jersey.version>
Expand Down
16 changes: 8 additions & 8 deletions hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ private void processTaskFinishedEvent(TaskFinishedEvent event) {
}
task.setFinishTime(event.getFinishTime());
task.setTaskStatus(getPre21Value(event.getTaskStatus()));
task.incorporateCounters(((TaskFinished) event.getDatum()).counters);
task.incorporateCounters(((TaskFinished) event.getDatum()).getCounters());
}

private void processTaskFailedEvent(TaskFailedEvent event) {
Expand All @@ -472,7 +472,7 @@ private void processTaskFailedEvent(TaskFailedEvent event) {
task.setFinishTime(event.getFinishTime());
task.setTaskStatus(getPre21Value(event.getTaskStatus()));
TaskFailed t = (TaskFailed)(event.getDatum());
task.putDiagnosticInfo(t.error.toString());
task.putDiagnosticInfo(t.getError().toString());
// killed task wouldn't have failed attempt.
if (t.getFailedDueToAttempt() != null) {
task.putFailedDueToAttemptId(t.getFailedDueToAttempt().toString());
Expand Down Expand Up @@ -542,7 +542,7 @@ private void processTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
}
attempt.setFinishTime(event.getFinishTime());
attempt
.incorporateCounters(((TaskAttemptFinished) event.getDatum()).counters);
.incorporateCounters(((TaskAttemptFinished) event.getDatum()).getCounters());
}

private void processReduceAttemptFinishedEvent(
Expand All @@ -568,7 +568,7 @@ private void processReduceAttemptFinishedEvent(
attempt.setShuffleFinished(event.getShuffleFinishTime());
attempt.setSortFinished(event.getSortFinishTime());
attempt
.incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters);
.incorporateCounters(((ReduceAttemptFinished) event.getDatum()).getCounters());
attempt.arraySetClockSplits(event.getClockSplits());
attempt.arraySetCpuUsages(event.getCpuUsages());
attempt.arraySetVMemKbytes(event.getVMemKbytes());
Expand Down Expand Up @@ -596,7 +596,7 @@ private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) {
// is redundant, but making this will add future-proofing.
attempt.setFinishTime(event.getFinishTime());
attempt
.incorporateCounters(((MapAttemptFinished) event.getDatum()).counters);
.incorporateCounters(((MapAttemptFinished) event.getDatum()).getCounters());
attempt.arraySetClockSplits(event.getClockSplits());
attempt.arraySetCpuUsages(event.getCpuUsages());
attempt.arraySetVMemKbytes(event.getVMemKbytes());
Expand Down Expand Up @@ -661,11 +661,11 @@ private void processJobFinishedEvent(JobFinishedEvent event) {

JobFinished job = (JobFinished)event.getDatum();
Map<String, Long> countersMap =
JobHistoryUtils.extractCounters(job.totalCounters);
JobHistoryUtils.extractCounters(job.getTotalCounters());
result.putTotalCounters(countersMap);
countersMap = JobHistoryUtils.extractCounters(job.mapCounters);
countersMap = JobHistoryUtils.extractCounters(job.getMapCounters());
result.putMapCounters(countersMap);
countersMap = JobHistoryUtils.extractCounters(job.reduceCounters);
countersMap = JobHistoryUtils.extractCounters(job.getReduceCounters());
result.putReduceCounters(countersMap);
}

Expand Down
6 changes: 3 additions & 3 deletions hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,9 @@ static boolean isJobConfXml(String fileName) {
static Map<String, Long> extractCounters(JhCounters counters) {
Map<String, Long> countersMap = new HashMap<String, Long>();
if (counters != null) {
for (JhCounterGroup group : counters.groups) {
for (JhCounter counter : group.counts) {
countersMap.put(counter.name.toString(), counter.value);
for (JhCounterGroup group : counters.getGroups()) {
for (JhCounter counter : group.getCounts()) {
countersMap.put(counter.getName().toString(), counter.getValue());
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,11 +268,11 @@ private static void incorporateCounter(SetField thunk, JhCounters counters,
String counterName) {
counterName = canonicalizeCounterName(counterName);

for (JhCounterGroup group : counters.groups) {
for (JhCounter counter : group.counts) {
for (JhCounterGroup group : counters.getGroups()) {
for (JhCounter counter : group.getCounts()) {
if (counterName
.equals(canonicalizeCounterName(counter.name.toString()))) {
thunk.set(counter.value);
.equals(canonicalizeCounterName(counter.getName().toString()))) {
thunk.set(counter.getValue());
return;
}
}
Expand Down
8 changes: 4 additions & 4 deletions hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -636,11 +636,11 @@ private static void incorporateCounter(SetField thunk, JhCounters counters,
String counterName) {
counterName = canonicalizeCounterName(counterName);

for (JhCounterGroup group : counters.groups) {
for (JhCounter counter : group.counts) {
for (JhCounterGroup group : counters.getGroups()) {
for (JhCounter counter : group.getCounts()) {
if (counterName
.equals(canonicalizeCounterName(counter.name.toString()))) {
thunk.set(counter.value);
.equals(canonicalizeCounterName(counter.getName().toString()))) {
thunk.set(counter.getValue());
return;
}
}
Expand Down

0 comments on commit a0a429c

Please sign in to comment.