Skip to content
This repository has been archived by the owner on Mar 26, 2020. It is now read-only.

Commit

Permalink
Default Options for new Volumes
Browse files Browse the repository at this point in the history 
- Added default group profiles for each volume
  types(replicate,disperse,distribute)
- Included perf xlators in volfile template and disabled in default
  profiles. So that these xlators can be enabled/disabled if
  required.
- Included "features/shard" xlator in client volfile(Fixes: #954)
- Enabled self heal by default for replicate and disperse volumes.
- On glusterd2 start/restart, group profiles are saved in
  `$workdir/templates/profiles.json` (Ex:
  `/var/lib/glusterd2/templates/profiles.json)
- To modify profile defaults, update respective default profile and
  restart glusterd2.(Note: Already created Volume options will not
  change, only applicable for new volumes)
- All perf xlators are disabled by default till we decide the best
  defaults.
- Disabled set/reset of default option profiles.

Signed-off-by: Aravinda VK <avishwan@redhat.com>
  • Loading branch information
@aravindavk
aravindavk authored and Atin Mukherjee committed Dec 11, 2018
1 parent 755eff7 commit 9361638
Show file tree
Hide file tree
Showing 8 changed files with 284 additions and 58 deletions.
63 changes: 61 additions & 2 deletions glusterd2/commands/volumes/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"

"github.com/gluster/glusterd2/glusterd2/brick"
"github.com/gluster/glusterd2/glusterd2/gdctx"
Expand All @@ -22,6 +25,7 @@ import (

"github.com/pborman/uuid"
log "github.com/sirupsen/logrus"
config "github.com/spf13/viper"
"golang.org/x/sys/unix"
)

Expand Down Expand Up @@ -302,8 +306,43 @@ func undoStoreVolume(c transaction.TxnCtx) error {
return storeVolInfo(c, "oldvolinfo")
}

// LoadDefaultGroupOptions loads the default group option map into the store
func LoadDefaultGroupOptions() error {
func loadDefaultGroupOptions() error {
defaultProfilesPath := path.Join(config.GetString("localstatedir"), "templates", "profiles.json")
// If directory not exists, create the directory and then generate default templates
_, err := os.Stat(defaultProfilesPath)
if os.IsNotExist(err) {
content, err := json.MarshalIndent(defaultGroupOptions, "", " ")
if err != nil {
return err
}

err = os.MkdirAll(path.Dir(defaultProfilesPath), os.ModeDir|os.ModePerm)
if err != nil {
return err
}
return ioutil.WriteFile(defaultProfilesPath, content, 0640)
} else if err == nil {
content, err := ioutil.ReadFile(defaultProfilesPath)
if err != nil {
return err
}
var grpOpts map[string]*api.OptionGroup
err = json.Unmarshal(content, &grpOpts)
if err != nil {
return err
}
defaultGroupOptions = grpOpts
return nil
}
return err
}

// InitDefaultGroupOptions loads the default group option map into the store
func InitDefaultGroupOptions() error {
err := loadDefaultGroupOptions()
if err != nil {
return err
}
groupOptions, err := json.Marshal(defaultGroupOptions)
if err != nil {
return err
Expand Down Expand Up @@ -380,3 +419,23 @@ func txnGenerateBrickVolfiles(c transaction.TxnCtx) error {
}
return nil
}

func containsReservedGroupProfile(opts interface{}) bool {
pfx := "profile.default."
switch value := opts.(type) {
case map[string]string:
for k := range value {
if strings.HasPrefix(k, pfx) {
return true
}
}
case []string:
for _, v := range value {
if strings.HasPrefix(v, pfx) {
return true
}
}
}

return false
}
192 changes: 137 additions & 55 deletions glusterd2/commands/volumes/grouped-options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,75 @@ import (

// defaultGroupOptions maps from a profile name to a set of options
var defaultGroupOptions = map[string]*api.OptionGroup{
"profile.gluster-block": {Name: "profile.gluster-block", Options: []api.VolumeOption{{Name: "performance/quick-read", OnValue: "off"},
{Name: "performance/read-ahead", OnValue: "off"},
{Name: "performance/io-cache", OnValue: "off"},
{Name: "performance/md-cache", OnValue: "off"},
{Name: "performance/open-behind", OnValue: "off"},
{Name: "performance/readdir-ahead", OnValue: "off"},
{Name: "performance/write-behind.strict-O_DIRECT", OnValue: "on"},
{Name: "protocol/client.filter-O_DIRECT", OnValue: "disable"},
{Name: "cluster/replicate.eager-lock", OnValue: "disable"},
{Name: "cluster/replicate.quorum-type", OnValue: "auto"},
{Name: "cluster/replicate.data-self-heal-algorithm", OnValue: "full"},
{Name: "cluster/replicate.locking-scheme", OnValue: "granular"},
{Name: "cluster/replicate.shd-max-threads", OnValue: "8"},
{Name: "cluster/replicate.shd-wait-qlength", OnValue: "10000"},
{Name: "features/shard", OnValue: "on"},
{Name: "features/shard.shard-block-size", OnValue: "64MB"},
{Name: "user.cifs", OnValue: "off"},
{Name: "protocol/server.rpc-auth-allow-insecure", OnValue: "on"}},
Description: "Enable this profile for optimal results, in block use case"},
"profile.SMB-small-file": {Name: "profile.SMB-small-file",
Options: []api.VolumeOption{{Name: "features/upcall.cache-invalidation", OnValue: "on"},
"profile.default.replicate": {
Name: "profile.default.replicate",
Options: []api.VolumeOption{
{Name: "cluster/replicate.self-heal-daemon", OnValue: "on"},
{Name: "performance/md-cache", OnValue: "off"},
{Name: "performance/open-behind", OnValue: "off"},
{Name: "performance/quick-read", OnValue: "off"},
{Name: "performance/io-cache", OnValue: "off"},
{Name: "performance/readdir-ahead", OnValue: "off"},
{Name: "performance/read-ahead", OnValue: "off"},
{Name: "performance/write-behind", OnValue: "off"},
},
Description: "Default volume options for Replicate or Distributed Replicate volumes",
},
"profile.default.disperse": {
Name: "profile.default.disperse",
Options: []api.VolumeOption{
{Name: "cluster/replicate.self-heal-daemon", OnValue: "on"},
{Name: "performance/md-cache", OnValue: "off"},
{Name: "performance/open-behind", OnValue: "off"},
{Name: "performance/quick-read", OnValue: "off"},
{Name: "performance/io-cache", OnValue: "off"},
{Name: "performance/readdir-ahead", OnValue: "off"},
{Name: "performance/read-ahead", OnValue: "off"},
{Name: "performance/write-behind", OnValue: "off"},
},
Description: "Default volume options for Disperse or Distributed Disperse volumes",
},
"profile.default.distribute": {
Name: "profile.default.distribute",
Options: []api.VolumeOption{
{Name: "performance/md-cache", OnValue: "off"},
{Name: "performance/open-behind", OnValue: "off"},
{Name: "performance/quick-read", OnValue: "off"},
{Name: "performance/io-cache", OnValue: "off"},
{Name: "performance/readdir-ahead", OnValue: "off"},
{Name: "performance/read-ahead", OnValue: "off"},
{Name: "performance/write-behind", OnValue: "off"},
},
Description: "Default volume options for Distribute volumes",
},
"profile.gluster-block": {
Name: "profile.gluster-block",
Options: []api.VolumeOption{
{Name: "performance/quick-read", OnValue: "off"},
{Name: "performance/read-ahead", OnValue: "off"},
{Name: "performance/io-cache", OnValue: "off"},
{Name: "performance/md-cache", OnValue: "off"},
{Name: "performance/open-behind", OnValue: "off"},
{Name: "performance/readdir-ahead", OnValue: "off"},
{Name: "performance/write-behind.strict-O_DIRECT", OnValue: "on"},
{Name: "protocol/client.filter-O_DIRECT", OnValue: "disable"},
{Name: "cluster/replicate.eager-lock", OnValue: "disable"},
{Name: "cluster/replicate.quorum-type", OnValue: "auto"},
{Name: "cluster/replicate.data-self-heal-algorithm", OnValue: "full"},
{Name: "cluster/replicate.locking-scheme", OnValue: "granular"},
{Name: "cluster/replicate.shd-max-threads", OnValue: "8"},
{Name: "cluster/replicate.shd-wait-qlength", OnValue: "10000"},
{Name: "features/shard", OnValue: "on"},
{Name: "features/shard.shard-block-size", OnValue: "64MB"},
{Name: "user.cifs", OnValue: "off"},
{Name: "protocol/server.rpc-auth-allow-insecure", OnValue: "on"},
},
Description: "Enable this profile for optimal results, in block use case",
},
"profile.SMB-small-file": {
Name: "profile.SMB-small-file",
Options: []api.VolumeOption{
{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "features/upcall.cache-invalidation-timeout", OnValue: "600"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"},
{Name: "performance/md-cache", OnValue: "on"},
Expand All @@ -42,10 +90,14 @@ var defaultGroupOptions = map[string]*api.OptionGroup{
{Name: "protocol/client.event-threads", OnValue: "4"},
{Name: "protocol/server.event-threads", OnValue: "4"},
{Name: "cluster/distribute.lookup-optimize", OnValue: "on"},
{Name: "cluster/distribute.readdir-optimize", OnValue: "on"}},
Description: "For use cases with dominant small file workload in SMB access, enable this profile"},
"profile.FUSE-small-file": {Name: "profile.FUSE-small-file",
Options: []api.VolumeOption{{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "cluster/distribute.readdir-optimize", OnValue: "on"},
},
Description: "For use cases with dominant small file workload in SMB access, enable this profile",
},
"profile.FUSE-small-file": {
Name: "profile.FUSE-small-file",
Options: []api.VolumeOption{
{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "features/upcall.cache-invalidation-timeout", OnValue: "600"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"},
{Name: "performance/md-cache", OnValue: "on"},
Expand All @@ -60,10 +112,14 @@ var defaultGroupOptions = map[string]*api.OptionGroup{
{Name: "protocol/client.event-threads", OnValue: "4"},
{Name: "protocol/server.event-threads", OnValue: "4"},
{Name: "cluster/distribute.lookup-optimize", OnValue: "on"},
{Name: "cluster/distribute.readdir-optimize", OnValue: "on"}},
Description: "For use cases with dominant small file workload in native FUSE mount access, enable this profile"},
"profile.SMB-large-file-EC": {Name: "profile.SMB-large-file-EC",
Options: []api.VolumeOption{{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "cluster/distribute.readdir-optimize", OnValue: "on"},
},
Description: "For use cases with dominant small file workload in native FUSE mount access, enable this profile",
},
"profile.SMB-large-file-EC": {
Name: "profile.SMB-large-file-EC",
Options: []api.VolumeOption{
{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "features/upcall.cache-invalidation-timeout", OnValue: "600"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"},
{Name: "performance/md-cache", OnValue: "on"},
Expand All @@ -80,10 +136,14 @@ var defaultGroupOptions = map[string]*api.OptionGroup{
{Name: "performance/write-behind.trickling-writes", OnValue: "off"},
{Name: "performance/write-behind.aggregate-size", OnValue: "1MB"},
{Name: "protocol/client.event-threads", OnValue: "4"},
{Name: "protocol/server.event-threads", OnValue: "4"}},
Description: "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with SMB access to an erasure coded volume"},
"profile.FUSE-large-file-EC": {Name: "profile.FUSE-large-file-EC",
Options: []api.VolumeOption{{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "protocol/server.event-threads", OnValue: "4"},
},
Description: "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with SMB access to an erasure coded volume",
},
"profile.FUSE-large-file-EC": {
Name: "profile.FUSE-large-file-EC",
Options: []api.VolumeOption{
{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "features/upcall.cache-invalidation-timeout", OnValue: "600"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"},
{Name: "performance.md-cache", OnValue: "on"},
Expand All @@ -99,26 +159,38 @@ var defaultGroupOptions = map[string]*api.OptionGroup{
{Name: "performance/write-behind.trickling-writes", OnValue: "off"},
{Name: "performance/write-behind.aggregate-size", OnValue: "1MB"},
{Name: "protocol/client.event-threads", OnValue: "4"},
{Name: "protocol/server.event-threads", OnValue: "4"}},
Description: "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with native FUSE mount access to an erasure coded volume"},
"profile.nl-cache": {Name: "profile.nl-cache",
Options: []api.VolumeOption{{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "protocol/server.event-threads", OnValue: "4"},
},
Description: "Enable this profile for use cases consisting of mostly large files like video surveillance, backup, video streaming and others, with native FUSE mount access to an erasure coded volume",
},
"profile.nl-cache": {
Name: "profile.nl-cache",
Options: []api.VolumeOption{
{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "features/upcall.cache-invalidation-timeout", OnValue: "600"},
{Name: "performance/nl-cache", OnValue: "on"},
{Name: "performance/nl-cache.nl-cache-timeout", OnValue: "600"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"}},
Description: "Enable this for the workloads that generate lots of lookups before creating files, eg: SMB access, git tool and others"},
"profile.metadata-cache": {Name: "profile.metadata-cache",
Options: []api.VolumeOption{{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"},
},
Description: "Enable this for the workloads that generate lots of lookups before creating files, eg: SMB access, git tool and others",
},
"profile.metadata-cache": {
Name: "profile.metadata-cache",
Options: []api.VolumeOption{
{Name: "features/upcall.cache-invalidation", OnValue: "on"},
{Name: "features/upcall.cache-invalidation-timeout", OnValue: "600"},
{Name: "performance/md-cache", OnValue: "on"},
{Name: "performance/quick-read.cache-invalidation", OnValue: "on"},
{Name: "performance/md-cache.cache-invalidation", OnValue: "on"},
{Name: "performance/md-cache.md-cache-timeout", OnValue: "600"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"}},
Description: "This profile enables metadata(stat, xattr) caching on client side. Its recommended to enable this for most workloads other than the ones that require 100% consistency like databases"},
"profile.virt": {Name: "profile.virt",
Options: []api.VolumeOption{{Name: "performance/quick-read", OnValue: "off"},
{Name: "protocol/server.inode-lru-limit", OnValue: "200000"},
},
Description: "This profile enables metadata(stat, xattr) caching on client side. Its recommended to enable this for most workloads other than the ones that require 100% consistency like databases",
},
"profile.virt": {
Name: "profile.virt",
Options: []api.VolumeOption{
{Name: "performance/quick-read", OnValue: "off"},
{Name: "performance/read-ahead", OnValue: "off"},
{Name: "performance/io-cache", OnValue: "off"},
{Name: "performance/io-threads.low-prio-threads", OnValue: "32"},
Expand All @@ -131,9 +203,12 @@ var defaultGroupOptions = map[string]*api.OptionGroup{
{Name: "cluster/replicate.shd-max-threads", OnValue: "8"},
{Name: "cluster/replicate.shd-wait-qlength", OnValue: "10000"},
{Name: "features/shard", OnValue: "on"},
{Name: "user.cifs", OnValue: "off"}},
Description: "Enable this profile, if the Gluster Volume is used to store virtual machines"},
"profile.db": {Name: "profile.db",
{Name: "user.cifs", OnValue: "off"},
},
Description: "Enable this profile, if the Gluster Volume is used to store virtual machines",
},
"profile.db": {
Name: "profile.db",
Options: []api.VolumeOption{
{Name: "performance/open-behind", OnValue: "on"},
{Name: "performance/write-behind", OnValue: "off"},
Expand All @@ -152,15 +227,22 @@ var defaultGroupOptions = map[string]*api.OptionGroup{
{Name: "performance/open-behind.read-after-open", OnValue: "on"},
{Name: "performance/write-behind.strict-O_DIRECT", OnValue: "on"},
},
Description: "Enable profile that tunes volume for database workload"},
"tls": {Name: "tls",
Description: "Enable profile that tunes volume for database workload",
},
"tls": {
Name: "tls",
Options: []api.VolumeOption{
{Name: "protocol/server.transport.socket.ssl-enabled", OnValue: "on"},
{Name: "protocol/client.transport.socket.ssl-enabled", OnValue: "on"},
},
Description: "Enable TLS for the volume for both bricks and clients"},
"profile.test": {Name: "profile.test",
Options: []api.VolumeOption{{Name: "cluster/replicate.eager-lock", OnValue: "on"},
{Name: "gfproxy.cluster/replicate.eager-lock", OnValue: "on"}},
Description: "Test purpose only"},
Description: "Enable TLS for the volume for both bricks and clients",
},
"profile.test": {
Name: "profile.test",
Options: []api.VolumeOption{
{Name: "cluster/replicate.eager-lock", OnValue: "on"},
{Name: "gfproxy.cluster/replicate.eager-lock", OnValue: "on"},
},
Description: "Test purpose only",
},
}
19 changes: 19 additions & 0 deletions glusterd2/commands/volumes/volume-create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) {
return
}

if containsReservedGroupProfile(req.Options) {
restutils.SendHTTPError(ctx, w, http.StatusBadRequest, gderrors.ErrReservedGroupProfile)
return
}

if req.Size > 0 {
applyDefaults(&req)

Expand Down Expand Up @@ -149,6 +154,20 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Include default Volume Options profile
if len(req.Subvols) > 0 {
groupProfile, exists := defaultGroupOptions["profile.default."+req.Subvols[0].Type]
if exists {
for _, opt := range groupProfile.Options {
// Apply default option only if not overridden in volume create request
_, exists = req.Options[opt.Name]
if !exists {
req.Options[opt.Name] = opt.OnValue
}
}
}
}

nodes, err := req.Nodes()
if err != nil {
restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err)
Expand Down
5 changes: 5 additions & 0 deletions glusterd2/commands/volumes/volume-option.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,11 @@ func volumeOptionsHandler(w http.ResponseWriter, r *http.Request) {
return
}

if containsReservedGroupProfile(req.Options) {
restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrReservedGroupProfile)
return
}

txn, err := transaction.NewTxnWithLocks(ctx, volname)
if err != nil {
status, err := restutils.ErrToStatusCode(err)
Expand Down
Loading

0 comments on commit 9361638

Please sign in to comment.