*: introduce an option UseLeaseTTL for lease manager's TTL

* Introduce an option UseLeaseTTL. Default is false.
* If UseLeaseTTL turned on, create lease manager with the TTL value.
* When gRPC turned on, remove previous workaround of setting leaseTTL
  to a huge value.
* Increase engine reconcile interval from 2 to 5 sec.

Originally written by Hector Fernandez <hectorj@gmail.com>
Taken from https://github.com/giantswarm/fleet/tree/patch_lease_ttl

Author: Dongsu Park

config/config.go

@@ -42,6 +42,7 @@ type Config struct {
 	UnitsDirectory          string
 	SystemdUser             bool
 	AuthorizedKeysFile      string
+	UseLeaseTTL             bool
 }
 
 func (c *Config) Capabilities() machine.Capabilities {

engine/engine.go

@@ -67,13 +67,6 @@ func New(reg CompleteRegistry, lManager lease.Manager, rStream pkg.EventStream,
 
 func (e *Engine) Run(ival time.Duration, stop <-chan struct{}) {
 	leaseTTL := ival * 5
-	if e.machine.State().Capabilities.Has(machine.CapGRPC) {
-		// With grpc it doesn't make sense to set to 5secs the TTL of the etcd key.
-		// This has a special impact whenever we have high worload in the cluster, cause
-		// it'd provoke constant leader re-elections.
-		// TODO: IMHO, this should be configurable via a flag to disable the TTL.
-		leaseTTL = ival * 500000
-	}
 	machID := e.machine.State().ID
 
 	reconcile := func() {

fleetd/fleetd.go

@@ -84,7 +84,7 @@ func Main() {
 	cfgset.String("etcd_cafile", "", "SSL Certificate Authority file used to secure etcd communication")
 	cfgset.String("etcd_key_prefix", registry.DefaultKeyPrefix, "Keyspace for fleet data in etcd")
 	cfgset.Float64("etcd_request_timeout", 1.0, "Amount of time in seconds to allow a single etcd request before considering it failed.")
-	cfgset.Float64("engine_reconcile_interval", 2.0, "Interval at which the engine should reconcile the cluster schedule in etcd.")
+	cfgset.Float64("engine_reconcile_interval", 5.0, "Interval at which the engine should reconcile the cluster schedule in etcd.")
 	cfgset.String("public_ip", "", "IP address that fleet machine should publish")
 	cfgset.String("metadata", "", "List of key-value metadata to assign to the fleet machine")
 	cfgset.String("agent_ttl", agent.DefaultTTL, "TTL in seconds of fleet machine state in etcd")
@@ -96,6 +96,7 @@ func Main() {
 	cfgset.Bool("disable_watches", false, "Disable the use of etcd watches. Increases scheduling latency")
 	cfgset.Bool("verify_units", false, "DEPRECATED - This option is ignored")
 	cfgset.String("authorized_keys_file", "", "DEPRECATED - This option is ignored")
+	cfgset.Bool("use_lease_ttl", false, "Enable the usage of TTL option when creating a lease key in etcd")
 
 	globalconf.Register("", cfgset)
 	cfg, err := getConfig(cfgset, *cfgPath)
@@ -237,6 +238,7 @@ func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error
 		SystemdUser:             (*flagset.Lookup("systemd_user")).Value.(flag.Getter).Get().(bool),
 		TokenLimit:              (*flagset.Lookup("token_limit")).Value.(flag.Getter).Get().(int),
 		AuthorizedKeysFile:      (*flagset.Lookup("authorized_keys_file")).Value.(flag.Getter).Get().(string),
+		UseLeaseTTL:             (*flagset.Lookup("use_lease_ttl")).Value.(flag.Getter).Get().(bool),
 	}
 
 	if cfg.VerifyUnits {

pkg/lease/etcd.go

@@ -20,6 +20,8 @@ import (
 	"time"
 
 	etcd "github.com/coreos/etcd/client"
+	"github.com/coreos/fleet/log"
+
 	"golang.org/x/net/context"
 )
 
@@ -53,8 +55,14 @@ func (l *etcdLease) Renew(period time.Duration) error {
 	val, err := serializeLeaseMetadata(l.meta.MachineID, l.meta.Version)
 	opts := &etcd.SetOptions{
 		PrevIndex: l.idx,
-		TTL:       period,
 	}
+
+	log.Infof("Renew %v", l.mgr.UseLeaseTTL)
+
+	if l.mgr.UseLeaseTTL {
+		opts.TTL = period
+	}
+
 	resp, err := l.mgr.kAPI.Set(l.mgr.ctx(), l.key, val, opts)
 	if err != nil {
 		return err
@@ -97,12 +105,13 @@ func serializeLeaseMetadata(machID string, ver int) (string, error) {
 }
 
 type etcdLeaseManager struct {
-	kAPI      etcd.KeysAPI
-	keyPrefix string
+	kAPI        etcd.KeysAPI
+	keyPrefix   string
+	UseLeaseTTL bool
 }
 
-func NewEtcdLeaseManager(kAPI etcd.KeysAPI, keyPrefix string) *etcdLeaseManager {
-	return &etcdLeaseManager{kAPI: kAPI, keyPrefix: keyPrefix}
+func NewEtcdLeaseManager(kAPI etcd.KeysAPI, keyPrefix string, useLeaseTTL bool) *etcdLeaseManager {
+	return &etcdLeaseManager{kAPI: kAPI, keyPrefix: keyPrefix, UseLeaseTTL: useLeaseTTL}
 }
 
 func (r *etcdLeaseManager) ctx() context.Context {
@@ -136,7 +145,12 @@ func (r *etcdLeaseManager) StealLease(name, machID string, ver int, period time.
 	key := r.leasePath(name)
 	opts := &etcd.SetOptions{
 		PrevIndex: idx,
-		TTL:       period,
+	}
+
+	log.Infof("StealLease %v", r.UseLeaseTTL)
+
+	if r.UseLeaseTTL {
+		opts.TTL = period
 	}
 	resp, err := r.kAPI.Set(r.ctx(), key, val, opts)
 	if err != nil {
@@ -158,9 +172,13 @@ func (r *etcdLeaseManager) AcquireLease(name string, machID string, ver int, per
 
 	key := r.leasePath(name)
 	opts := &etcd.SetOptions{
-		TTL:       period,
 		PrevExist: etcd.PrevNoExist,
 	}
+	log.Infof("StealLease %v", r.UseLeaseTTL)
+
+	if r.UseLeaseTTL {
+		opts.TTL = period
+	}
 
 	resp, err := r.kAPI.Set(r.ctx(), key, val, opts)
 	if err != nil {

server/server.go

@@ -110,7 +110,7 @@ func New(cfg config.Config, listeners []net.Listener) (*Server, error) {
 		reg        engine.CompleteRegistry
 		genericReg interface{}
 	)
-	lManager := lease.NewEtcdLeaseManager(kAPI, cfg.EtcdKeyPrefix)
+	lManager := lease.NewEtcdLeaseManager(kAPI, cfg.EtcdKeyPrefix, cfg.UseLeaseTTL)
 
 	if !cfg.EnableGRPC {
 		genericReg = registry.NewEtcdRegistry(kAPI, cfg.EtcdKeyPrefix)