Skip to content

Commit

Permalink
[PLAT-3555]: Added an option to retain time-unit for Schedules.
Browse files Browse the repository at this point in the history
Summary:
Added a field to retain the scheduler's frequency and backup expiry time unit.
We will be keeping Minutes as default for existing schedules and Days for backup expiry time.
Also, allow users to change it while editing schedules.

Test Plan:
Verified manually by creating the backup and schedules using various input time unit.
Added Unit test case.

Reviewers: vkumar, vpatibandla

Reviewed By: vpatibandla

Subscribers: jenkins-bot, sanketh, yugaware

Differential Revision: https://phabricator.dev.yugabyte.com/D16517
  • Loading branch information
vipul-yb committed Apr 19, 2022
1 parent ea7c07a commit b5482b4
Show file tree
Hide file tree
Showing 14 changed files with 263 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ public void run() {
tableBackupParams.sse = params().sse;
tableBackupParams.parallelism = params().parallelism;
tableBackupParams.timeBeforeDelete = params().timeBeforeDelete;
tableBackupParams.expiryTimeUnit = params().expiryTimeUnit;
tableBackupParams.backupType = params().backupType;
tableBackupParams.isFullBackup = CollectionUtils.isEmpty(params().keyspaceTableList);
Set<String> tablesToBackup = new HashSet<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@ public Result createBackupYb(UUID customerUUID) {
backupUtil.validateTables(null, universe, null, taskParams.backupType);
}

if (taskParams.timeBeforeDelete != 0L && taskParams.expiryTimeUnit == null) {
throw new PlatformServiceException(BAD_REQUEST, "Please provide time unit for backup expiry");
}

if (taskParams.storageConfigUUID == null) {
throw new PlatformServiceException(
BAD_REQUEST, "Missing StorageConfig UUID: " + taskParams.storageConfigUUID);
Expand Down Expand Up @@ -273,6 +277,10 @@ public Result createBackupSchedule(UUID customerUUID) {
BAD_REQUEST, "Cannot provide both Cron Expression and Scheduling frequency");
} else if (taskParams.schedulingFrequency != 0L) {
BackupUtil.validateBackupFrequency(taskParams.schedulingFrequency);
if (taskParams.frequencyTimeUnit == null) {
throw new PlatformServiceException(
BAD_REQUEST, "Please provide time unit for scheduler frequency");
}
} else if (taskParams.cronExpression != null) {
BackupUtil.validateBackupCronExpression(taskParams.cronExpression);
}
Expand Down Expand Up @@ -304,11 +312,12 @@ public Result createBackupSchedule(UUID customerUUID) {
Schedule.create(
customerUUID,
taskParams.universeUUID,
taskParams.scheduleName,
taskParams,
TaskType.CreateBackup,
taskParams.schedulingFrequency,
taskParams.cronExpression);
taskParams.cronExpression,
taskParams.frequencyTimeUnit,
taskParams.scheduleName);
UUID scheduleUUID = schedule.getScheduleUUID();
LOG.info(
"Created backup schedule for customer {}, schedule uuid = {}.", customerUUID, scheduleUUID);
Expand Down Expand Up @@ -665,10 +674,13 @@ public Result editBackup(UUID customerUUID, UUID backupUUID) {
throw new PlatformServiceException(
BAD_REQUEST,
"Please provide either a positive expiry time or storage config to edit backup");
}
if (Backup.IN_PROGRESS_STATES.contains(backup.state)) {
} else if (Backup.IN_PROGRESS_STATES.contains(backup.state)) {
throw new PlatformServiceException(
BAD_REQUEST, "Cannot edit a backup that is in progress state");
} else if (taskParams.timeBeforeDeleteFromPresentInMillis > 0L
&& taskParams.expiryTimeUnit == null) {
throw new PlatformServiceException(
BAD_REQUEST, "Please provide a time unit for backup expiry");
}
if (taskParams.storageConfigUUID != null) {
updateBackupStorageConfig(customerUUID, backupUUID, taskParams);
Expand All @@ -677,6 +689,7 @@ public Result editBackup(UUID customerUUID, UUID backupUUID) {
}
if (taskParams.timeBeforeDeleteFromPresentInMillis > 0L) {
backup.updateExpiryTime(taskParams.timeBeforeDeleteFromPresentInMillis);
backup.updateExpiryTimeUnit(taskParams.expiryTimeUnit);
LOG.info(
"Updated Backup {} expiry time before delete to {} ms",
backupUUID,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,12 @@ public Result editBackupSchedule(UUID customerUUID, UUID scheduleUUID) {
} else if (schedule.getStatus().equals(State.Active) && schedule.getRunningState()) {
throw new PlatformServiceException(CONFLICT, "Cannot edit schedule as it is running.");
} else if (params.frequency != null) {
if (params.frequencyTimeUnit == null) {
throw new PlatformServiceException(BAD_REQUEST, "Please provide time unit for frequency");
}
BackupUtil.validateBackupFrequency(params.frequency);
schedule.updateFrequency(params.frequency);
schedule.updateFrequencyTimeUnit(params.frequencyTimeUnit);
} else if (params.cronExpression != null) {
BackupUtil.validateBackupCronExpression(params.cronExpression);
schedule.updateCronExpression(params.cronExpression);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.yugabyte.yw.common.Util;
import com.yugabyte.yw.models.helpers.TimeUnit;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.HashSet;
Expand Down Expand Up @@ -35,6 +36,9 @@ public class BackupRequestParams extends UniverseTaskParams {
@ApiModelProperty(value = "Time before deleting the backup from storage, in milliseconds")
public long timeBeforeDelete = 0L;

@ApiModelProperty(value = "Time unit for user input schedule frequency")
public TimeUnit frequencyTimeUnit;

// Should backup script enable verbose logging.
@ApiModelProperty(value = "Is verbose logging enabled")
public boolean enableVerboseLogs = false;
Expand Down Expand Up @@ -81,6 +85,9 @@ public class BackupRequestParams extends UniverseTaskParams {
@ApiModelProperty(value = "Minimum number of backups to retain for a particular backup schedule")
public int minNumBackupsToRetain = Util.MIN_NUM_BACKUPS_TO_RETAIN;

@ApiModelProperty(value = "Time unit for backup expiry time")
public TimeUnit expiryTimeUnit;

@ApiModel(description = "Keyspace and table info for backup")
public static class KeyspaceTable {
@ApiModelProperty(value = "Tables")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.yugabyte.yw.common.BackupUtil;
import com.yugabyte.yw.common.Util;
import com.yugabyte.yw.models.Backup.StorageConfigType;
import com.yugabyte.yw.models.helpers.TimeUnit;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.HashSet;
Expand Down Expand Up @@ -122,6 +123,9 @@ public enum ActionType {
@ApiModelProperty(value = "Type of backup storage config")
public StorageConfigType storageConfigType = null;

@ApiModelProperty(value = "Time unit for backup expiry time")
public TimeUnit expiryTimeUnit = TimeUnit.DAYS;

@JsonIgnore
public Set<String> getTableNames() {
Set<String> tableNames = new HashSet<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.yugabyte.yw.forms;

import com.yugabyte.yw.models.helpers.TimeUnit;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.UUID;
Expand All @@ -12,4 +13,7 @@ public class EditBackupParams {

@ApiModelProperty(value = "New backup Storage config")
public UUID storageConfigUUID = null;

@ApiModelProperty(value = "Time unit for backup expiry")
public TimeUnit expiryTimeUnit;
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.yugabyte.yw.forms;

import com.yugabyte.yw.models.Schedule.State;
import com.yugabyte.yw.models.helpers.TimeUnit;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;

Expand All @@ -15,4 +16,7 @@ public class EditBackupScheduleParams {

@ApiModelProperty(value = "Cron expression for scheduling")
public String cronExpression;

@ApiModelProperty(value = "Time Unit for frequency")
public TimeUnit frequencyTimeUnit;
}
19 changes: 19 additions & 0 deletions managed/src/main/java/com/yugabyte/yw/models/Backup.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.yugabyte.yw.forms.BackupTableParams;
import com.yugabyte.yw.models.filters.BackupFilter;
import com.yugabyte.yw.models.helpers.TaskType;
import com.yugabyte.yw.models.helpers.TimeUnit;
import com.yugabyte.yw.models.paging.BackupPagedApiResponse;
import com.yugabyte.yw.models.paging.BackupPagedQuery;
import com.yugabyte.yw.models.paging.BackupPagedResponse;
Expand Down Expand Up @@ -210,6 +211,23 @@ public void updateExpiryTime(long timeBeforeDeleteFromPresent) {
save();
}

@ApiModelProperty(value = "Time unit for backup expiry time", accessMode = READ_WRITE)
@Column
private TimeUnit expiryTimeUnit;

public TimeUnit getExpiryTimeUnit() {
return this.expiryTimeUnit;
}

public void setExpiryTimeUnit(TimeUnit expiryTimeUnit) {
this.expiryTimeUnit = expiryTimeUnit;
}

public void updateExpiryTimeUnit(TimeUnit expiryTimeUnit) {
setExpiryTimeUnit(expiryTimeUnit);
save();
}

public void updateStorageConfigUUID(UUID storageConfigUUID) {
this.storageConfigUUID = storageConfigUUID;
this.backupInfo.storageConfigUUID = storageConfigUUID;
Expand Down Expand Up @@ -277,6 +295,7 @@ public static Backup create(
}
if (params.timeBeforeDelete != 0L) {
backup.expiry = new Date(System.currentTimeMillis() + params.timeBeforeDelete);
backup.setExpiryTimeUnit(params.expiryTimeUnit);
}
if (params.backupList != null) {
params.backupUuid = backup.backupUUID;
Expand Down
47 changes: 42 additions & 5 deletions managed/src/main/java/com/yugabyte/yw/models/Schedule.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.yugabyte.yw.forms.ITaskParams;
import com.yugabyte.yw.models.filters.ScheduleFilter;
import com.yugabyte.yw.models.helpers.TaskType;
import com.yugabyte.yw.models.helpers.TimeUnit;
import com.yugabyte.yw.models.paging.PagedQuery;
import com.yugabyte.yw.models.paging.SchedulePagedQuery;
import com.yugabyte.yw.models.paging.SchedulePagedResponse;
Expand Down Expand Up @@ -108,7 +109,7 @@ public int getFailureCount() {
return failureCount;
}

@ApiModelProperty(value = "Frequency of the schedule, in minutes", accessMode = READ_WRITE)
@ApiModelProperty(value = "Frequency of the schedule, in milli seconds", accessMode = READ_WRITE)
@Column(nullable = false)
private long frequency;

Expand Down Expand Up @@ -164,6 +165,17 @@ public String getScheduleName() {
@Column(nullable = false)
private UUID ownerUUID;

@ApiModelProperty(value = "Time unit of frequency", accessMode = READ_WRITE)
private TimeUnit frequencyTimeUnit;

public TimeUnit getFrequencyTimeUnit() {
return this.frequencyTimeUnit;
}

public void setFrequencyTimeunit(TimeUnit frequencyTimeUnit) {
this.frequencyTimeUnit = frequencyTimeUnit;
}

public String getCronExpression() {
return cronExpression;
}
Expand Down Expand Up @@ -208,6 +220,11 @@ public void updateCronExpression(String cronExpression) {
resetSchedule();
}

public void updateFrequencyTimeUnit(TimeUnit frequencyTimeUnit) {
setFrequencyTimeunit(frequencyTimeUnit);
save();
}

@Column(nullable = false)
@ApiModelProperty(value = "Running state of the schedule")
private boolean runningState = false;
Expand All @@ -226,11 +243,12 @@ public void setRunningState(boolean state) {
public static Schedule create(
UUID customerUUID,
UUID ownerUUID,
String scheduleName,
ITaskParams params,
TaskType taskType,
long frequency,
String cronExpression) {
String cronExpression,
TimeUnit frequencyTimeUnit,
String scheduleName) {
Schedule schedule = new Schedule();
schedule.scheduleUUID = UUID.randomUUID();
schedule.customerUUID = customerUUID;
Expand All @@ -241,6 +259,7 @@ public static Schedule create(
schedule.status = State.Active;
schedule.cronExpression = cronExpression;
schedule.ownerUUID = ownerUUID;
schedule.frequencyTimeUnit = frequencyTimeUnit;
schedule.scheduleName =
scheduleName != null ? scheduleName : "schedule-" + schedule.scheduleUUID;
schedule.save();
Expand All @@ -252,13 +271,31 @@ public static Schedule create(
ITaskParams params,
TaskType taskType,
long frequency,
String cronExpression) {
String cronExpression,
TimeUnit frequencyTimeUnit) {
UUID ownerUUID = customerUUID;
JsonNode scheduleParams = Json.toJson(params);
if (scheduleParams.has("universeUUID")) {
ownerUUID = UUID.fromString(scheduleParams.get("universeUUID").asText());
}
return create(customerUUID, ownerUUID, null, params, taskType, frequency, cronExpression);
return create(
customerUUID,
ownerUUID,
params,
taskType,
frequency,
cronExpression,
frequencyTimeUnit,
null);
}

public static Schedule create(
UUID customerUUID,
ITaskParams params,
TaskType taskType,
long frequency,
String cronExpression) {
return create(customerUUID, params, taskType, frequency, cronExpression, TimeUnit.MINUTES);
}

/** DEPRECATED: use {@link #getOrBadRequest()} */
Expand Down
41 changes: 41 additions & 0 deletions managed/src/main/java/com/yugabyte/yw/models/helpers/TimeUnit.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright 2022 YugaByte, Inc. and Contributors
*
* Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt
*/
package com.yugabyte.yw.models.helpers;

import io.ebean.annotation.EnumValue;

public enum TimeUnit {
@EnumValue("NanoSeconds")
NANOSECONDS,

@EnumValue("MicroSeconds")
MICROSECONDS,

@EnumValue("MilliSeconds")
MILLISECONDS,

@EnumValue("Seconds")
SECONDS,

@EnumValue("Minutes")
MINUTES,

@EnumValue("Hours")
HOURS,

@EnumValue("Days")
DAYS,

@EnumValue("Months")
MONTHS,

@EnumValue("Years")
YEARS
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- Copyright (c) YugaByte, Inc.

ALTER TABLE backup ADD COLUMN IF NOT EXISTS expiry_time_unit varchar(50);
UPDATE backup SET expiry_time_unit = 'Days' WHERE expiry IS NOT NULL;
ALTER TABLE backup DROP CONSTRAINT IF EXISTS ck_expiry_time_unit;
ALTER TABLE backup ADD CONSTRAINT ck_expiry_time_unit CHECK (expiry_time_unit in ('NanoSeconds','MicroSeconds','MilliSeconds','Seconds','Minutes','Hours', 'Days', 'Months', 'Years'));

ALTER TABLE schedule ADD COLUMN IF NOT EXISTS frequency_time_unit varchar(50) DEFAULT 'Minutes';
ALTER TABLE schedule DROP CONSTRAINT IF EXISTS ck_frequency_time_unit;
ALTER TABLE schedule ADD CONSTRAINT ck_frequency_time_unit CHECK (frequency_time_unit in ('NanoSeconds','MicroSeconds','MilliSeconds','Seconds','Minutes','Hours', 'Days', 'Months', 'Years'));
Loading

0 comments on commit b5482b4

Please sign in to comment.