From db7f6749762e5869f8efd5ae982ee418c7c819dd Mon Sep 17 00:00:00 2001 From: Gianluca Arbezzano Date: Tue, 14 Mar 2017 23:49:53 +0000 Subject: [PATCH 1/3] Bootstrap autodetection and swarm zero conf Fixed #9 This PR bootstrap the autodetection feature: 1. It works when the configuration file is not setup. Right know I am not going to manage the merge of double sources (autodetection and configuration file). 2. At the moment only Docker Swarm will support this feature. --- cmd/daemon.go | 29 ++++++++++++++++++----------- core/autodetect.go | 21 +++++++++++++++++++++ core/daemon.go | 2 +- core/daemon_test.go | 6 +++--- 4 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 core/autodetect.go diff --git a/cmd/daemon.go b/cmd/daemon.go index 4398cc7..1577f85 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -25,7 +25,7 @@ func (c *DaemonCmd) Run(args []string) int { var debug bool cmdFlags := flag.NewFlagSet("event", flag.ExitOnError) cmdFlags.StringVar(&port, "port", ":8000", "port") - cmdFlags.StringVar(&configPath, "config", "/etc/deamon.yml", "config") + cmdFlags.StringVar(&configPath, "config", "", "config") cmdFlags.BoolVar(&debug, "debug", false, "debug") if err := cmdFlags.Parse(args); err != nil { logrus.WithField("error", err).Warn("Problem to parse arguments.") @@ -35,15 +35,22 @@ func (c *DaemonCmd) Run(args []string) int { logrus.SetLevel(logrus.DebugLevel) logrus.Debug("Daemon started in debug mode") } - config, err := readConfiguration(configPath) - if err != nil { - logrus.WithField("error", err).Warn("Configuration file malformed.") - return 1 - } - core, err := core.NewCore(config.AutoscalersConf) - if err != nil { - logrus.WithField("error", err).Warn(err) - return 1 + var coreEngine core.Core + if configPath != "" { + config, err := readConfiguration(configPath) + if err != nil { + logrus.WithField("error", err).Warn("Configuration file malformed.") + return 1 + } + logrus.Infof("Starting from configuration file located %s", configPath) + coreEngine, err = core.NewCoreByConfig(config.AutoscalersConf) + if err != nil { + logrus.WithField("error", err).Warn(err) + return 1 + } + } else { + logrus.Info("Starting in auto-detection mode.") + coreEngine, err = core.Autodetect() } go func() { sigchan := make(chan os.Signal, 10) @@ -52,7 +59,7 @@ func (c *DaemonCmd) Run(args []string) int { logrus.Info("Stopping and cleaning. Bye!") os.Exit(0) }() - router := api.GetRouter(core, c.EventChannel) + router := api.GetRouter(coreEngine, c.EventChannel) logrus.Infof("API Server run on port %s", port) http.ListenAndServe(port, router) return 0 diff --git a/core/autodetect.go b/core/autodetect.go new file mode 100644 index 0000000..d719629 --- /dev/null +++ b/core/autodetect.go @@ -0,0 +1,21 @@ +package core + +import "github.com/gianarb/orbiter/autoscaler" + +// This function use diferent strategies to get information from +// the system itself to configure the autoloader. +// They can be environment variables for example or other systems. +func Autodetect() (Core error) { + scalers := autoscaler.Autoscalers{} + var core Core + autoDetectSwarmMode(&scalers) + return core, nil +} + +func autoDetectSwarmMode(a *autoscaler.Autoscalers) { + // Create Docker Client by EnvVar and check if it's working. + + // Get List of Services + + // Check which services has labels and register them to orbiter. +} diff --git a/core/daemon.go b/core/daemon.go index 1fefe66..cf3b750 100644 --- a/core/daemon.go +++ b/core/daemon.go @@ -11,7 +11,7 @@ type Core struct { Autoscalers autoscaler.Autoscalers } -func NewCore(c map[string]AutoscalerConf) (Core, error) { +func NewCoreByConfig(c map[string]AutoscalerConf) (Core, error) { scalers := autoscaler.Autoscalers{} var core Core for scalerName, scaler := range c { diff --git a/core/daemon_test.go b/core/daemon_test.go index 5a588d1..c4f824f 100644 --- a/core/daemon_test.go +++ b/core/daemon_test.go @@ -29,7 +29,7 @@ func TestNewCore(t *testing.T) { }, }, } - core, err := NewCore(conf) + core, err := NewCoreByConfig(conf) if err != nil { t.Fatal(err) } @@ -55,7 +55,7 @@ func TestGetSingleAutoscaler(t *testing.T) { }, }, } - core, _ := NewCore(conf) + core, _ := NewCoreByConfig(conf) _, ok := core.Autoscalers["second/micro"] if ok == false { t.Fatal("micro exist") @@ -89,7 +89,7 @@ func TestNewCoreWithUnsupportedProvider(t *testing.T) { }, }, } - _, err := NewCore(conf) + _, err := NewCoreByConfig(conf) if err.Error() != "lalala not supported." { t.Fatal(err) } From 4e841fe6cc38c50f00137ca1699cf652c0de2ba8 Mon Sep 17 00:00:00 2001 From: Gianluca Arbezzano Date: Wed, 15 Mar 2017 21:36:48 +0000 Subject: [PATCH 2/3] Builded autotedection for swarm. --- cmd/daemon.go | 20 +++++++---- core/autodetect.go | 84 +++++++++++++++++++++++++++++++++++++++------- core/daemon.go | 11 +++--- 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/cmd/daemon.go b/cmd/daemon.go index 1577f85..afa83d3 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -11,6 +11,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/gianarb/orbiter/api" + "github.com/gianarb/orbiter/autoscaler" "github.com/gianarb/orbiter/core" ) @@ -29,28 +30,35 @@ func (c *DaemonCmd) Run(args []string) int { cmdFlags.BoolVar(&debug, "debug", false, "debug") if err := cmdFlags.Parse(args); err != nil { logrus.WithField("error", err).Warn("Problem to parse arguments.") - return 1 + os.Exit(1) } if debug == true { logrus.SetLevel(logrus.DebugLevel) logrus.Debug("Daemon started in debug mode") } - var coreEngine core.Core + coreEngine := core.Core{ + Autoscalers: autoscaler.Autoscalers{}, + } + var err error if configPath != "" { config, err := readConfiguration(configPath) if err != nil { logrus.WithField("error", err).Warn("Configuration file malformed.") - return 1 + os.Exit(1) } logrus.Infof("Starting from configuration file located %s", configPath) - coreEngine, err = core.NewCoreByConfig(config.AutoscalersConf) + err = core.NewCoreByConfig(config.AutoscalersConf, &coreEngine) if err != nil { logrus.WithField("error", err).Warn(err) - return 1 + os.Exit(1) } } else { logrus.Info("Starting in auto-detection mode.") - coreEngine, err = core.Autodetect() + err = core.Autodetect(&coreEngine) + if err != nil { + logrus.WithField("error", err).Info(err) + os.Exit(0) + } } go func() { sigchan := make(chan os.Signal, 10) diff --git a/core/autodetect.go b/core/autodetect.go index d719629..dcd448e 100644 --- a/core/autodetect.go +++ b/core/autodetect.go @@ -1,21 +1,81 @@ package core -import "github.com/gianarb/orbiter/autoscaler" +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/client" + "github.com/gianarb/orbiter/autoscaler" + "github.com/gianarb/orbiter/provider" +) // This function use diferent strategies to get information from // the system itself to configure the autoloader. // They can be environment variables for example or other systems. -func Autodetect() (Core error) { - scalers := autoscaler.Autoscalers{} - var core Core - autoDetectSwarmMode(&scalers) - return core, nil +func Autodetect(core *Core) error { + autoDetectSwarmMode(core) + if len(core.Autoscalers) == 0 { + return errors.New("we didn't detect any autoscaling group") + } + return nil } -func autoDetectSwarmMode(a *autoscaler.Autoscalers) { - // Create Docker Client by EnvVar and check if it's working. - - // Get List of Services - - // Check which services has labels and register them to orbiter. +func autoDetectSwarmMode(c *Core) { + ctx := context.Background() + dockerClient, err := client.NewEnvClient() + if err != nil { + logrus.WithField("error", err).Debug("Problem communication with Docker") + return + } + info, err := dockerClient.Info(ctx) + if err != nil { + logrus.WithField("error", err).Debug("We didn't detect any Docker Swarm running") + return + } + if info.Swarm.NodeID == "" { + logrus.Debug("We didn't detect any Docker Swarm running") + return + } + services, err := dockerClient.ServiceList(ctx, types.ServiceListOptions{}) + if err != nil { + logrus.WithField("error", err).Debug("Bad comunication with Docker.") + return + } + prov, _ := provider.NewSwarmProvider(map[string]string{}) + for _, service := range services { + s, err := getAutoscalerByService(prov, service.Spec.Annotations) + if err != nil { + continue + } + logrus.Debugf("autodetect_swarm/%s added to orbiter.", service.Spec.Annotations.Name) + c.Autoscalers[fmt.Sprintf("autodetect_swarm/%s", service.Spec.Annotations.Name)] = s + } +} + +func getAutoscalerByService(p autoscaler.Provider, an swarm.Annotations) (autoscaler.Autoscaler, error) { + _, e := an.Labels["orbiter"] + if e == false { + return autoscaler.Autoscaler{}, errors.New("") + } + up := convertStringLabelToInt("orbiter.up", an.Labels) + down := convertStringLabelToInt("orbiter.down", an.Labels) + as := autoscaler.NewAutoscaler(p, an.Name, up, down) + return as, nil +} + +func convertStringLabelToInt(labelName string, labels map[string]string) int { + row, e := labels[labelName] + if e == false { + i, err := strconv.ParseInt(row, 10, 64) + if err != nil { + return 1 + } + return int(i) + } + return 1 } diff --git a/core/daemon.go b/core/daemon.go index cf3b750..c1b9632 100644 --- a/core/daemon.go +++ b/core/daemon.go @@ -11,20 +11,17 @@ type Core struct { Autoscalers autoscaler.Autoscalers } -func NewCoreByConfig(c map[string]AutoscalerConf) (Core, error) { +func NewCoreByConfig(c map[string]AutoscalerConf, core *Core) error { scalers := autoscaler.Autoscalers{} - var core Core for scalerName, scaler := range c { p, err := provider.NewProvider(scaler.Provider, scaler.Parameters) if err != nil { - return core, err + return err } for serviceName, policy := range scaler.Policies { scalers[fmt.Sprintf("%s/%s", scalerName, serviceName)] = autoscaler.NewAutoscaler(p, serviceName, policy.Up, policy.Down) } } - core = Core{ - Autoscalers: scalers, - } - return core, nil + core.Autoscalers = scalers + return nil } From 3ce4c171f10e5d2ec8af6401da5ce7c19bfd32a1 Mon Sep 17 00:00:00 2001 From: Gianluca Arbezzano Date: Wed, 15 Mar 2017 21:56:58 +0000 Subject: [PATCH 3/3] Fixed tests --- core/daemon_test.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/core/daemon_test.go b/core/daemon_test.go index c4f824f..a48868a 100644 --- a/core/daemon_test.go +++ b/core/daemon_test.go @@ -1,8 +1,15 @@ package core -import "testing" +import ( + "testing" + + "github.com/gianarb/orbiter/autoscaler" +) func TestNewCore(t *testing.T) { + core := Core{ + Autoscalers: autoscaler.Autoscalers{}, + } conf := map[string]AutoscalerConf{ "first-scaler": AutoscalerConf{ Provider: "fake", @@ -29,7 +36,7 @@ func TestNewCore(t *testing.T) { }, }, } - core, err := NewCoreByConfig(conf) + err := NewCoreByConfig(conf, &core) if err != nil { t.Fatal(err) } @@ -39,6 +46,9 @@ func TestNewCore(t *testing.T) { } func TestGetSingleAutoscaler(t *testing.T) { + core := Core{ + Autoscalers: autoscaler.Autoscalers{}, + } conf := map[string]AutoscalerConf{ "second": AutoscalerConf{ Provider: "fake", @@ -55,7 +65,7 @@ func TestGetSingleAutoscaler(t *testing.T) { }, }, } - core, _ := NewCoreByConfig(conf) + NewCoreByConfig(conf, &core) _, ok := core.Autoscalers["second/micro"] if ok == false { t.Fatal("micro exist") @@ -63,6 +73,9 @@ func TestGetSingleAutoscaler(t *testing.T) { } func TestNewCoreWithUnsupportedProvider(t *testing.T) { + core := Core{ + Autoscalers: autoscaler.Autoscalers{}, + } conf := map[string]AutoscalerConf{ "second-scaler": AutoscalerConf{ Provider: "fake", @@ -89,7 +102,7 @@ func TestNewCoreWithUnsupportedProvider(t *testing.T) { }, }, } - _, err := NewCoreByConfig(conf) + err := NewCoreByConfig(conf, &core) if err.Error() != "lalala not supported." { t.Fatal(err) }