alihan/fx-serverless
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor kubeconfig persist logic (#412)

Browse Source 
* add Dir function to config

* clean up

* persist the kubeconf on the fly

Signed-off-by: Minghe Huang <h.minghe@gmail.com>

* fix test

Signed-off-by: Minghe Huang <h.minghe@gmail.com>

* simplicity the cloud config fetch

Signed-off-by: Minghe Huang <h.minghe@gmail.com>
This commit is contained in:
Minghe
2019-12-18 10:58:34 +08:00
committed by GitHub
parent d868ebf4a1
commit 1f7714c1e9
7 changed files with 159 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
config/config.go
View File

@@ -7,6 +7,7 @@ import (
"os"
"os/user"
"path"
"path/filepath"
dockerInfra "github.com/metrue/fx/infra/docker"
"github.com/metrue/fx/types"
@@ -22,6 +23,7 @@ type Configer interface {
UseCloud(name string) error
View() ([]byte, error)
AddCloud(name string, meta []byte) error
Dir() (string, error)
}
// Config config of fx
@@ -191,6 +193,15 @@ func (c *Config) writeDefaultConfig() error {
return c.UseCloud("default")
}
// Dir get directory of config
func (c *Config) Dir() (string, error) {
p, err := filepath.Abs(c.configFile)
if err != nil {
return "", err
}
return path.Dir(p), nil
}
var (
_ Configer = &Config{}
)

62
config/config_test.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"os"
"os/user"
"path/filepath"
"reflect"
"testing"
@@ -49,32 +50,39 @@ func TestConfig(t *testing.T) {
t.Fatalf("should get %s but got %s", "default", cloudMeta["name"])
}
// add k8s cloud
kCloud := k8sInfra.Cloud{
Type: types.CloudTypeK8S,
Config: "sample kubeconfg",
Token: "",
URL: "",
Nodes: map[string]k8sInfra.Noder{
"master-node": &k8sInfra.Node{
IP: "1.1.1.1",
User: "user-1",
Type: "k3s-master",
Name: "master-node",
},
"agent-node-1": &k8sInfra.Node{
IP: "1.1.1.1",
User: "user-1",
Type: "k3s-agent",
Name: "agent-node-1",
},
},
n1, err := k8sInfra.CreateNode(
"1.1.1.1",
"user-1",
"k3s-master",
"master-node",
)
if err != nil {
t.Fatal(err)
}
n2, err := k8sInfra.CreateNode(
"1.1.1.1",
"user-1",
"k3s-agent",
"agent-node-1",
)
if err != nil {
t.Fatal(err)
}
kName := "k8s-1"
kubeconf := "./tmp/" + kName + "config.yml"
defer func() {
if err := os.RemoveAll(kubeconf); err != nil {
t.Fatal(err)
}
}()
// add k8s cloud
kCloud := k8sInfra.NewCloud(kubeconf, n1, n2)
kMeta, err := kCloud.Dump()
if err != nil {
t.Fatal(err)
}
kName := "k8s-1"
if err := c.AddCloud(kName, kMeta); err != nil {
t.Fatal(err)
}
@@ -108,4 +116,16 @@ func TestConfig(t *testing.T) {
t.Fatal(err)
}
fmt.Println(string(body))
dir, err := c.Dir()
if err != nil {
t.Fatal(err)
}
here, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
if dir != filepath.Join(here, "./tmp") {
t.Fatalf("should get %s but got %s", "./tmp", dir)
}
}

2
fx.go
View File

@@ -16,7 +16,7 @@ import (
"github.com/urfave/cli"
)
const version = "0.8.74"
const version = "0.8.75"
func init() {
go checkForUpdate()

12
handlers/infra.go
View File

@@ -2,6 +2,7 @@ package handlers
import (
"fmt"
"path/filepath"
"strings"
"github.com/metrue/fx/config"
@@ -11,7 +12,7 @@ import (
"github.com/metrue/fx/pkg/spinner"
)
func setupK8S(masterInfo string, agentsInfo string) ([]byte, error) {
func setupK8S(configDir string, name, masterInfo string, agentsInfo string) ([]byte, error) {
info := strings.Split(masterInfo, "@")
if len(info) != 2 {
return nil, fmt.Errorf("incorrect master info, should be <user>@<ip> format")
@@ -36,7 +37,8 @@ func setupK8S(masterInfo string, agentsInfo string) ([]byte, error) {
nodes = append(nodes, node)
}
}
cloud := k8sInfra.NewCloud(nodes...)
kubeconfigPath := filepath.Join(configDir, name+".kubeconfig")
cloud := k8sInfra.NewCloud(kubeconfigPath, nodes...)
if err := cloud.Provision(); err != nil {
return nil, err
}
@@ -91,7 +93,11 @@ func Setup(ctx context.Contexter) (err error) {
switch strings.ToLower(typ) {
case "k8s":
kubeconf, err := setupK8S(cli.String("master"), cli.String("agents"))
dir, err := fxConfig.Dir()
if err != nil {
return err
}
kubeconf, err := setupK8S(dir, name, cli.String("master"), cli.String("agents"))
if err != nil {
return err
}

83
infra/k8s/cloud.go
View File

@@ -3,18 +3,22 @@ package k8s
import (
"encoding/json"
"fmt"
"io/ioutil"
"github.com/metrue/fx/infra"
"github.com/metrue/fx/types"
"github.com/metrue/fx/utils"
)
// Cloud define a cloud
type Cloud struct {
Config string `json:"config"`
URL string `json:"url"`
Token string `json:"token"`
Type string `json:"type"`
Nodes map[string]Noder `json:"nodes"`
// Define where is the location of kubeconf would be saved to
KubeConfig string `json:"config"`
Type string `json:"type"`
Nodes map[string]Noder `json:"nodes"`
token string
url string
}
// Load a cloud from config
@@ -39,14 +43,16 @@ func Load(meta []byte, messup ...func(n Noder) (Noder, error)) (*Cloud, error) {
}
// NewCloud new a cloud
func NewCloud(node ...Noder) *Cloud {
func NewCloud(kubeconf string, node ...Noder) *Cloud {
nodes := map[string]Noder{}
for _, n := range node {
nodes[n.GetName()] = n
}
return &Cloud{
Type: types.CloudTypeK8S,
Nodes: nodes,
KubeConfig: kubeconf,
Type: types.CloudTypeK8S,
Nodes: nodes,
}
}
@@ -64,7 +70,7 @@ func (c *Cloud) Provision() error {
// when it's k3s cluster
if master != nil {
c.URL = fmt.Sprintf("https://%s:6443", master.GetIP())
c.url = fmt.Sprintf("https://%s:6443", master.GetIP())
if err := master.Provision(map[string]string{}); err != nil {
return err
}
@@ -73,22 +79,19 @@ func (c *Cloud) Provision() error {
if err != nil {
return err
}
c.Token = tok
c.token = tok
config, err := master.GetConfig()
if err != nil {
return err
}
c.Config = config
}
// when it's a docker agent
if len(agents) == 1 && agents[0].GetType() == NodeTypeDocker {
config, err := agents[0].GetConfig()
if err != nil {
if err := utils.EnsureFile(c.KubeConfig); err != nil {
return err
}
if err := ioutil.WriteFile(c.KubeConfig, []byte(config), 0666); err != nil {
return err
}
c.Config = config
}
if len(agents) > 0 {
@@ -98,8 +101,8 @@ func (c *Cloud) Provision() error {
for _, agent := range agents {
go func(node Noder) {
errCh <- node.Provision(map[string]string{
"url": c.URL,
"token": c.Token,
"url": c.url,
"token": c.token,
})
}(agent)
}
@@ -118,8 +121,8 @@ func (c *Cloud) Provision() error {
func (c *Cloud) AddNode(n Noder, skipProvision bool) error {
if !skipProvision {
if err := n.Provision(map[string]string{
"url": c.URL,
"token": c.Token,
"url": c.url,
"token": c.token,
}); err != nil {
return err
}
@@ -173,16 +176,16 @@ func (c *Cloud) UnmarshalJSON(data []byte) error {
} else if k == "token" {
tok, ok := v.(string)
if ok {
c.Token = tok
c.token = tok
} else {
c.Token = ""
c.token = ""
}
} else if k == "config" {
config, ok := v.(string)
if ok {
c.Config = config
c.KubeConfig = config
} else {
c.Config = ""
c.KubeConfig = ""
}
} else if k == "type" {
typ, ok := v.(string)
@@ -194,9 +197,9 @@ func (c *Cloud) UnmarshalJSON(data []byte) error {
} else if k == "url" {
url, ok := v.(string)
if ok {
c.URL = url
c.url = url
} else {
c.URL = ""
c.url = ""
}
}
}
@@ -217,17 +220,17 @@ func (c *Cloud) MarshalJSON() ([]byte, error) {
}
body, err := json.Marshal(struct {
URL string `json:"url"`
Config string `json:"config"`
Type string `json:"type"`
Token string `json:"token"`
Nodes map[string]Node `json:"nodes"`
URL string `json:"url"`
KubeConfig string `json:"config"`
Type string `json:"type"`
Token string `json:"token"`
Nodes map[string]Node `json:"nodes"`
}{
URL: c.URL,
Config: c.Config,
Type: c.Type,
Token: c.Token,
Nodes: nodes,
KubeConfig: c.KubeConfig,
Type: c.Type,
Token: c.token,
URL: c.url,
Nodes: nodes,
})
if err != nil {
return nil, err
@@ -248,13 +251,13 @@ func (c *Cloud) Dump() ([]byte, error) {
// GetConfig get config
func (c *Cloud) GetConfig() (string, error) {
if c.Config != "" {
return c.Config, nil
if c.KubeConfig != "" {
return c.KubeConfig, nil
}
if err := c.Provision(); err != nil {
return "", err
}
return c.Config, nil
return c.KubeConfig, nil
}
// IsHealth check if cloud is in health

63
infra/k8s/cloud_test.go
View File

@@ -3,6 +3,8 @@ package k8s
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/golang/mock/gomock"
@@ -25,6 +27,13 @@ func TestLoad(t *testing.T) {
})
t.Run("only master node", func(t *testing.T) {
kubeconfig := "./kubeconfig.yml"
defer func() {
if err := os.RemoveAll("./kubeconfig.yml"); err != nil {
t.Fatal(err)
}
}()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -36,18 +45,19 @@ func TestLoad(t *testing.T) {
name := "master"
ip := "127.0.0.1"
user := "testuser"
kubeconfContent := "sample-content"
master.EXPECT().GetName().Return(name)
master.EXPECT().GetType().Return(typ).Times(2)
master.EXPECT().GetIP().Return(ip).Times(2)
master.EXPECT().GetUser().Return(user)
master.EXPECT().GetConfig().Return("sample-config", nil)
master.EXPECT().GetConfig().Return(kubeconfContent, nil)
claud := &Cloud{
Config: "",
URL: "",
Token: "",
Type: "k8s",
Nodes: map[string]Noder{"master-node": master},
KubeConfig: kubeconfig,
Type: "k8s",
url: "",
token: "",
Nodes: map[string]Noder{"master-node": master},
}
meta, err := json.Marshal(claud)
@@ -67,9 +77,24 @@ func TestLoad(t *testing.T) {
if err := cloud.Provision(); err != nil {
t.Fatal(err)
}
content, err := ioutil.ReadFile(claud.KubeConfig)
if err != nil {
t.Fatal(err)
}
if string(content) != kubeconfContent {
t.Fatalf("should get %s but got %s", kubeconfContent, content)
}
})
t.Run("one master node and one agent", func(t *testing.T) {
kubeconfig := "./kubeconfig.yml"
defer func() {
if err := os.RemoveAll("./kubeconfig.yml"); err != nil {
t.Fatal(err)
}
}()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -85,10 +110,11 @@ func TestLoad(t *testing.T) {
name := "master"
ip := "127.0.0.1"
user := "testuser"
kubeconfContent := "sample-config"
master.EXPECT().GetName().Return(name)
master.EXPECT().GetType().Return(typ).Times(2)
master.EXPECT().GetIP().Return(ip).Times(3)
master.EXPECT().GetConfig().Return("sample-config", nil)
master.EXPECT().GetConfig().Return(kubeconfContent, nil)
master.EXPECT().GetUser().Return(user)
nodeType := NodeTypeAgent
@@ -96,18 +122,18 @@ func TestLoad(t *testing.T) {
nodeIP := "12.12.12.12"
nodeUser := "testuser"
node.EXPECT().GetName().Return(nodeName)
node.EXPECT().GetType().Return(nodeType).Times(3)
node.EXPECT().GetType().Return(nodeType).Times(2)
node.EXPECT().GetIP().Return(nodeIP)
node.EXPECT().GetUser().Return(nodeUser)
url := fmt.Sprintf("https://%s:6443", master.GetIP())
tok := "tok-1"
claud := &Cloud{
Config: "",
URL: url,
Token: tok,
Type: "k8s",
Nodes: map[string]Noder{"master-node": master, "agent-node": node},
KubeConfig: kubeconfig,
url: url,
token: tok,
Type: "k8s",
Nodes: map[string]Noder{"master-node": master, "agent-node": node},
}
meta, err := json.Marshal(claud)
if err != nil {
@@ -125,12 +151,19 @@ func TestLoad(t *testing.T) {
master.EXPECT().Provision(map[string]string{}).Return(nil)
master.EXPECT().GetToken().Return(tok, nil)
node.EXPECT().Provision(map[string]string{
"url": cloud.URL,
"token": cloud.Token,
"url": cloud.url,
"token": cloud.token,
}).Return(nil)
if err := cloud.Provision(); err != nil {
t.Fatal(err)
}
content, err := ioutil.ReadFile(claud.KubeConfig)
if err != nil {
t.Fatal(err)
}
if string(content) != kubeconfContent {
t.Fatalf("should get %s but got %s", kubeconfContent, content)
}
})
}

17
middlewares/provision.go
View File

@@ -56,13 +56,7 @@ func Provision(ctx context.Contexter) (err error) {
return err
}
var deployer infra.Deployer
if os.Getenv("KUBECONFIG") != "" {
deployer, err = k8sInfra.CreateDeployer(os.Getenv("KUBECONFIG"))
if err != nil {
return err
}
ctx.Set("cloud_type", types.CloudTypeK8S)
} else if cloud.GetType() == types.CloudTypeDocker {
if cloud.GetType() == types.CloudTypeDocker {
var meta map[string]string
if err := json.Unmarshal([]byte(conf), &meta); err != nil {
return err
@@ -79,11 +73,12 @@ func Provision(ctx context.Contexter) (err error) {
return err
}
} else if cloud.GetType() == types.CloudTypeK8S {
kubeconfig, err := fxConfig.GetKubeConfig()
if err != nil {
return err
ctx.Set("cloud_type", types.CloudTypeK8S)
if os.Getenv("KUBECONFIG") != "" {
conf = os.Getenv("KUBECONFIG")
}
deployer, err = k8sInfra.CreateDeployer(kubeconfig)
deployer, err = k8sInfra.CreateDeployer(conf)
if err != nil {
return err
}