From 8af1788515dc9cc19742aa8078c3c9b832f1f197 Mon Sep 17 00:00:00 2001 From: agam Date: Tue, 13 Oct 2026 11:06:48 +0530 Subject: [PATCH] [GOBBLIN-2271] Report retention counts per dataset DatasetMetric (GaaSJobObservabilityEvent.datasetsMetrics) models only writes, so a data-retention job cannot report per-dataset deletions: the counts it puts in datasetTaskSummaries are dropped when GaaS parses them into DatasetTaskSummary. Add optional versionsTotal, versionsDeleted and directoriesPruned (nullable long, default null) to DatasetMetric and DatasetTaskSummary, and map them in toDatasetMetric. Existing producers and consumers are unaffected; the 5-arg DatasetTaskSummary constructor is unchanged. --- .../main/avro/GaaSJobObservabilityEvent.avsc | 18 +++++++ .../gobblin/runtime/DatasetTaskSummary.java | 18 ++++++- .../runtime/DatasetTaskSummaryTest.java | 53 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 gobblin-runtime/src/test/java/org/apache/gobblin/runtime/DatasetTaskSummaryTest.java diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSJobObservabilityEvent.avsc b/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSJobObservabilityEvent.avsc index 165560f9be9..08822c239d1 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSJobObservabilityEvent.avsc +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSJobObservabilityEvent.avsc @@ -276,6 +276,24 @@ "type": ["null","string"], "doc": "Comma-separated list of the partitions committed for the dataset in this run, null if the writer does not commit partitions or does not report them", "default": null + }, + { + "name": "versionsTotal", + "type": ["null","long"], + "doc": "Retention (data-deletion) jobs only: number of dataset versions considered under the dataset root (deletable + retained), null for jobs that do not delete", + "default": null + }, + { + "name": "versionsDeleted", + "type": ["null","long"], + "doc": "Retention (data-deletion) jobs only: number of dataset versions deleted (or, on a dry run, that would be deleted), null for jobs that do not delete", + "default": null + }, + { + "name": "directoriesPruned", + "type": ["null","long"], + "doc": "Retention (data-deletion) jobs only: number of now-empty parent directories pruned after the versions were deleted, null for jobs that do not delete", + "default": null } ] } diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java index adc8a15159e..087601cc14b 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java @@ -51,11 +51,27 @@ public class DatasetTaskSummary { // run can commit more than one snapshot/partition); null = unsupported/unreported. private String snapshotsCommitted; private String partitionsCommitted; + // Retention (data-deletion) counts, likewise not @NonNull and populated during JSON deserialization of + // events that carry them (e.g. DDM retention runs); null = not a deletion job. + private Long versionsTotal; + private Long versionsDeleted; + private Long directoriesPruned; /** * Convert a {@link DatasetTaskSummary} to a {@link DatasetMetric}. */ public static DatasetMetric toDatasetMetric(DatasetTaskSummary datasetTaskSummary) { - return new DatasetMetric(datasetTaskSummary.getDatasetUrn(), datasetTaskSummary.getBytesWritten(), datasetTaskSummary.getRecordsWritten(), datasetTaskSummary.isSuccessfullyCommitted(), datasetTaskSummary.getDataQualityStatus(), datasetTaskSummary.getSnapshotsCommitted(), datasetTaskSummary.getPartitionsCommitted()); + return DatasetMetric.newBuilder() + .setDatasetUrn(datasetTaskSummary.getDatasetUrn()) + .setBytesWritten(datasetTaskSummary.getBytesWritten()) + .setEntitiesWritten(datasetTaskSummary.getRecordsWritten()) + .setSuccessfullyCommitted(datasetTaskSummary.isSuccessfullyCommitted()) + .setDataQualityStatus(datasetTaskSummary.getDataQualityStatus()) + .setSnapshotsCommitted(datasetTaskSummary.getSnapshotsCommitted()) + .setPartitionsCommitted(datasetTaskSummary.getPartitionsCommitted()) + .setVersionsTotal(datasetTaskSummary.getVersionsTotal()) + .setVersionsDeleted(datasetTaskSummary.getVersionsDeleted()) + .setDirectoriesPruned(datasetTaskSummary.getDirectoriesPruned()) + .build(); } } diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/DatasetTaskSummaryTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/DatasetTaskSummaryTest.java new file mode 100644 index 00000000000..c1d661f8b2f --- /dev/null +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/DatasetTaskSummaryTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime; + +import java.lang.reflect.Type; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.google.gson.reflect.TypeToken; + +import org.apache.gobblin.metrics.DatasetMetric; +import org.apache.gobblin.runtime.util.GsonUtils; + + +public class DatasetTaskSummaryTest { + + private static final Type SUMMARY_LIST_TYPE = new TypeToken>() { }.getType(); + + @Test + public void testToDatasetMetricMapsRetentionCountsFromJson() { + // The shape a DDM retention run emits: nothing written, deletion counts set. + String json = "[{\"datasetUrn\":\"hdfs://nn/data/foo\",\"recordsWritten\":0,\"bytesWritten\":0," + + "\"successfullyCommitted\":true,\"versionsTotal\":10,\"versionsDeleted\":4,\"directoriesPruned\":1}]"; + List summaries = GsonUtils.GSON_WITH_DATE_HANDLING.fromJson(json, SUMMARY_LIST_TYPE); + + DatasetMetric metric = DatasetTaskSummary.toDatasetMetric(summaries.get(0)); + + Assert.assertEquals(metric.getDatasetUrn(), "hdfs://nn/data/foo"); + Assert.assertEquals(metric.getEntitiesWritten(), Long.valueOf(0L)); + Assert.assertEquals(metric.getBytesWritten(), Long.valueOf(0L)); + Assert.assertTrue(metric.getSuccessfullyCommitted()); + Assert.assertEquals(metric.getVersionsTotal(), Long.valueOf(10L)); + Assert.assertEquals(metric.getVersionsDeleted(), Long.valueOf(4L)); + Assert.assertEquals(metric.getDirectoriesPruned(), Long.valueOf(1L)); + } +}