diff --git a/container_runtimes/docker/http/api.go b/container_runtimes/docker/http/api.go index 4d50286b..4b11ef35 100644 --- a/container_runtimes/docker/http/api.go +++ b/container_runtimes/docker/http/api.go @@ -129,6 +129,41 @@ func (api *API) post(path string, body []byte, expectStatus int, v interface{}) return nil } +// Version get version of docker engine +func (api *API) Version(ctx context.Context) (string, error) { + path := api.endpoint + "/version" + if !strings.HasPrefix(path, "http") { + path = "http://" + path + } + + req, err := http.NewRequest("GET", path, nil) + if err != nil { + return "", err + } + client := &http.Client{Timeout: 20 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", err + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("request %s failed: %d - %s", path, resp.StatusCode, resp.Status) + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var res dockerTypes.Version + err = json.Unmarshal(body, &res) + if err != nil { + return "", err + } + return res.APIVersion, nil +} + // ListContainer list service func (api *API) ListContainer(ctx context.Context, name string) ([]types.Service, error) { if name != "" { diff --git a/container_runtimes/docker/sdk/docker.go b/container_runtimes/docker/sdk/docker.go index d83df163..fdd0a358 100644 --- a/container_runtimes/docker/sdk/docker.go +++ b/container_runtimes/docker/sdk/docker.go @@ -235,6 +235,15 @@ func (d *Docker) ListContainer(ctx context.Context, name string) ([]types.Servic return services, nil } +// Version get version of docker engine +func (d *Docker) Version(ctx context.Context) (string, error) { + ping, err := d.Ping(ctx) + if err != nil { + return "", err + } + return ping.APIVersion, nil +} + var ( _ containerruntimes.ContainerRuntime = &Docker{} ) diff --git a/container_runtimes/runtimes.go b/container_runtimes/runtimes.go index 9f65ddf9..ebf96077 100644 --- a/container_runtimes/runtimes.go +++ b/container_runtimes/runtimes.go @@ -16,4 +16,5 @@ type ContainerRuntime interface { StopContainer(ctx context.Context, name string) error InspectContainer(ctx context.Context, name string, container interface{}) error ListContainer(ctx context.Context, filter string) ([]types.Service, error) + Version(ctx context.Context) (string, error) } diff --git a/deploy/deployer.go b/deploy/deployer.go index 08e31ab9..211c51bf 100644 --- a/deploy/deployer.go +++ b/deploy/deployer.go @@ -13,4 +13,5 @@ type Deployer interface { Update(ctx context.Context, name string) error GetStatus(ctx context.Context, name string) (types.Service, error) List(ctx context.Context, name string) ([]types.Service, error) + Ping(ctx context.Context) error } diff --git a/deploy/docker/docker.go b/deploy/docker/docker.go index 763a6fd9..63feef98 100644 --- a/deploy/docker/docker.go +++ b/deploy/docker/docker.go @@ -76,6 +76,14 @@ func (d *Docker) GetStatus(ctx context.Context, name string) (types.Service, err return service, nil } +// Ping check healty status of infra +func (d *Docker) Ping(ctx context.Context) error { + if _, err := d.cli.Version(ctx); err != nil { + return err + } + return nil +} + // List services func (d *Docker) List(ctx context.Context, name string) ([]types.Service, error) { // FIXME support remote host diff --git a/deploy/k3s/constants.go b/deploy/k3s/constants.go deleted file mode 100644 index a5d8eeeb..00000000 --- a/deploy/k3s/constants.go +++ /dev/null @@ -1,8 +0,0 @@ -package k3s - -// ConfigMap is the key to function docker project source code in configmap -var ConfigMap = struct { - AppMetaEnvName string -}{ - AppMetaEnvName: "APP_META", -} diff --git a/deploy/k3s/deployment.go b/deploy/k3s/deployment.go deleted file mode 100644 index c348b406..00000000 --- a/deploy/k3s/deployment.go +++ /dev/null @@ -1,102 +0,0 @@ -package k3s - -import ( - "fmt" - - "github.com/metrue/fx/types" - appsv1 "k8s.io/api/apps/v1" - apiv1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func generateDeploymentSpec( - name string, - image string, - bindPorts []types.PortBinding, - replicas int32, - selector map[string]string, -) *appsv1.Deployment { - ports := []apiv1.ContainerPort{} - for index, binding := range bindPorts { - ports = append(ports, apiv1.ContainerPort{ - Name: fmt.Sprintf("fx-container-%d", index), - ContainerPort: binding.ContainerExposePort, - }) - } - - container := apiv1.Container{ - Name: "fx-placeholder-container-name", - Image: image, - Ports: ports, - ImagePullPolicy: v1.PullIfNotPresent, - } - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: selector, - }, - Template: apiv1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: selector, - }, - Spec: apiv1.PodSpec{ - Containers: []apiv1.Container{container}, - }, - }, - }, - } -} - -// GetDeployment get a deployment -func (k *K3S) GetDeployment(namespace string, name string) (*appsv1.Deployment, error) { - return k.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{}) -} - -// CreateDeployment create a deployment -func (k *K3S) CreateDeployment( - namespace string, - name string, - image string, - ports []types.PortBinding, - replicas int32, - selector map[string]string, -) (*appsv1.Deployment, error) { - deployment := generateDeploymentSpec(name, image, ports, replicas, selector) - return k.AppsV1().Deployments(namespace).Create(deployment) -} - -// UpdateDeployment update a deployment -func (k *K3S) UpdateDeployment( - namespace string, - name string, - image string, - ports []types.PortBinding, - replicas int32, - selector map[string]string, -) (*appsv1.Deployment, error) { - deployment := generateDeploymentSpec(name, image, ports, replicas, selector) - return k.AppsV1().Deployments(namespace).Update(deployment) -} - -// DeleteDeployment delete a deployment -func (k *K3S) DeleteDeployment(namespace string, name string) error { - return k.AppsV1().Deployments(namespace).Delete(name, &metav1.DeleteOptions{}) -} - -// CreateDeploymentWithInitContainer create a deployment which will wait InitContainer to do the image build before function container start -func (k *K3S) CreateDeploymentWithInitContainer( - namespace string, - name string, - ports []types.PortBinding, - replicas int32, - selector map[string]string, -) (*appsv1.Deployment, error) { - deployment := generateDeploymentSpec(name, name, ports, replicas, selector) - updatedDeployment := injectInitContainer(name, deployment) - return k.AppsV1().Deployments(namespace).Create(updatedDeployment) -} diff --git a/deploy/k3s/deployment_test.go b/deploy/k3s/deployment_test.go deleted file mode 100644 index 286fba68..00000000 --- a/deploy/k3s/deployment_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package k3s - -import ( - "os" - "testing" - - "github.com/metrue/fx/types" -) - -func TestDeployment(t *testing.T) { - namespace := "default" - name := "fx-hello-world" - image := "metrue/kube-hello" - selector := map[string]string{ - "app": "fx-app", - } - - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - t.Skip("skip test since no KUBECONFIG given in environment variable") - } - - k8s, err := Create("") - if err != nil { - t.Fatal(err) - } - if _, err := k8s.GetDeployment(namespace, name); err == nil { - t.Fatalf("should get not found error") - } - - replicas := int32(2) - bindings := []types.PortBinding{ - types.PortBinding{ - ServiceBindingPort: 80, - ContainerExposePort: 3000, - }, - types.PortBinding{ - ServiceBindingPort: 443, - ContainerExposePort: 3000, - }, - } - deployment, err := k8s.CreateDeployment(namespace, name, image, bindings, replicas, selector) - if err != nil { - t.Fatal(err) - } - if deployment == nil { - t.Fatalf("deploymetn should not be %v", nil) - } - - if deployment.Name != name { - t.Fatalf("should get %s but got %s", name, deployment.Name) - } - - if *deployment.Spec.Replicas != replicas { - t.Fatalf("should get %v but got %v", replicas, deployment.Spec.Replicas) - } - - if err := k8s.DeleteDeployment(namespace, name); err != nil { - t.Fatal(err) - } -} diff --git a/deploy/k3s/init_container.go b/deploy/k3s/init_container.go deleted file mode 100644 index d0b76025..00000000 --- a/deploy/k3s/init_container.go +++ /dev/null @@ -1,70 +0,0 @@ -package k3s - -import ( - appsv1 "k8s.io/api/apps/v1" - apiv1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" -) - -// This is docker image provided by fx/contrib/docker_packer -// it can build a Docker image with give Docker project source codes encoded with base64 -// check the detail fx/contrib/docker_packer/main.go -const image = "metrue/fx-docker" - -func injectInitContainer(name string, deployment *appsv1.Deployment) *appsv1.Deployment { - configMapHasToBeReady := true - valueInConfigMapHasToBeReady := true - initContainer := v1.Container{ - Name: "fx-docker-build-c", - Image: image, - ImagePullPolicy: v1.PullAlways, - Command: []string{ - "/bin/sh", - "-c", - "/usr/bin/docker_packer $(APP_META) " + name, - }, // Maybe it can be passed by Binary data from config map - // Args: []string{"${APP_META}"}, // function source codes and name - VolumeMounts: []v1.VolumeMount{ - v1.VolumeMount{ - Name: "dockersock", - MountPath: "/var/run/docker.sock", - }, - }, - Env: []v1.EnvVar{ - v1.EnvVar{ - Name: ConfigMap.AppMetaEnvName, - ValueFrom: &v1.EnvVarSource{ - ConfigMapKeyRef: &v1.ConfigMapKeySelector{ - LocalObjectReference: v1.LocalObjectReference{Name: name}, - Key: ConfigMap.AppMetaEnvName, - Optional: &valueInConfigMapHasToBeReady, - }, - }, - }, - }, - EnvFrom: []v1.EnvFromSource{ - v1.EnvFromSource{ - ConfigMapRef: &v1.ConfigMapEnvSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: name, - }, - Optional: &configMapHasToBeReady, - }, - }, - }, - } - - volumes := []v1.Volume{ - v1.Volume{ - Name: "dockersock", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/run/docker.sock", - }, - }, - }, - } - deployment.Spec.Template.Spec.InitContainers = []apiv1.Container{initContainer} - deployment.Spec.Template.Spec.Volumes = volumes - return deployment -} diff --git a/deploy/k3s/k3s.go b/deploy/k3s/k3s.go deleted file mode 100644 index b94c308a..00000000 --- a/deploy/k3s/k3s.go +++ /dev/null @@ -1,151 +0,0 @@ -package k3s - -import ( - "context" - "os" - - "github.com/metrue/fx/deploy" - "github.com/metrue/fx/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -// K3S client -type K3S struct { - *kubernetes.Clientset -} - -const namespace = "default" - -// Create a k8s cluster client -func Create(kubeconfig string) (*K3S, error) { - if os.Getenv("KUBECONFIG") != "" { - kubeconfig = os.Getenv("KUBECONFIG") - } - - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return nil, err - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - return &K3S{clientset}, nil -} - -// Deploy a image to be a service -func (k *K3S) Deploy( - ctx context.Context, - fn types.Func, - name string, - image string, - ports []types.PortBinding, -) error { - selector := map[string]string{ - "app": "fx-app-" + name, - } - - const replicas = int32(3) - if _, err := k.GetDeployment(namespace, name); err != nil { - // TODO enable passing replica from fx CLI - if _, err := k.CreateDeployment( - namespace, - name, - image, - ports, - replicas, - selector, - ); err != nil { - return err - } - } else { - if _, err := k.UpdateDeployment( - namespace, - name, - image, - ports, - replicas, - selector, - ); err != nil { - return err - } - } - - // TODO fx should be able to know what's the target Kubernetes service platform - // it's going to deploy to - typ := "LoadBalancer" - if os.Getenv("SERVICE_TYPE") != "" { - typ = os.Getenv("SERVICE_TYPE") - } - - if _, err := k.GetService(namespace, name); err != nil { - if _, err := k.CreateService( - namespace, - name, - typ, - ports, - selector, - ); err != nil { - return err - } - } else { - if _, err := k.UpdateService( - namespace, - name, - typ, - ports, - selector, - ); err != nil { - return err - } - } - return nil -} - -// Update a service -func (k *K3S) Update(ctx context.Context, name string) error { - return nil -} - -// Destroy a service -func (k *K3S) Destroy(ctx context.Context, name string) error { - if err := k.DeleteService(namespace, name); err != nil { - return err - } - if err := k.DeleteDeployment(namespace, name); err != nil { - return err - } - return nil -} - -// GetStatus get status of a service -func (k *K3S) GetStatus(ctx context.Context, name string) (types.Service, error) { - svc, err := k.GetService(namespace, name) - service := types.Service{} - if err != nil { - return service, err - } - - service.Host = svc.Spec.ClusterIP - if len(svc.Spec.ExternalIPs) > 0 { - service.Host = svc.Spec.ExternalIPs[0] - } - - for _, port := range svc.Spec.Ports { - // TODO should clearify which port (target port, node port) should use - service.Port = int(port.Port) - break - } - return service, nil -} - -// List services -func (k *K3S) List(ctx context.Context, name string) ([]types.Service, error) { - return []types.Service{}, nil -} - -var ( - _ deploy.Deployer = &K3S{} -) diff --git a/deploy/k3s/service.go b/deploy/k3s/service.go deleted file mode 100644 index 0fb0b8b1..00000000 --- a/deploy/k3s/service.go +++ /dev/null @@ -1,87 +0,0 @@ -package k3s - -import ( - "strconv" - - "github.com/metrue/fx/types" - apiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - intstr "k8s.io/apimachinery/pkg/util/intstr" -) - -func generateServiceSpec( - namespace string, - name string, - typ string, - bindings []types.PortBinding, - selector map[string]string, -) *apiv1.Service { - servicePorts := []apiv1.ServicePort{} - for index, binding := range bindings { - servicePorts = append(servicePorts, apiv1.ServicePort{ - Name: "port-" + strconv.Itoa(index), - Protocol: apiv1.ProtocolTCP, - Port: binding.ServiceBindingPort, - TargetPort: intstr.FromInt(int(binding.ContainerExposePort)), - }) - } - - return &apiv1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - ClusterName: namespace, - }, - Spec: apiv1.ServiceSpec{ - Ports: servicePorts, - Type: apiv1.ServiceType(typ), - Selector: selector, - }, - } -} - -// CreateService create a service -func (k *K3S) CreateService( - namespace string, - name string, - typ string, - bindings []types.PortBinding, - selector map[string]string, -) (*apiv1.Service, error) { - service := generateServiceSpec(namespace, name, typ, bindings, selector) - createdService, err := k.CoreV1().Services(namespace).Create(service) - if err != nil { - return nil, err - } - - return createdService, nil -} - -// UpdateService update a service -// TODO this method is not perfect yet, should refactor later -func (k *K3S) UpdateService( - namespace string, - name string, - typ string, - bindings []types.PortBinding, - selector map[string]string, -) (*apiv1.Service, error) { - svc, err := k.GetService(namespace, name) - if err != nil { - return nil, err - } - svc.Spec.Selector = selector - svc.Spec.Type = apiv1.ServiceType(typ) - return k.CoreV1().Services(namespace).Update(svc) -} - -// DeleteService a service -func (k *K3S) DeleteService(namespace string, name string) error { - // TODO figure out the elegant way to delete a service - options := &metav1.DeleteOptions{} - return k.CoreV1().Services(namespace).Delete(name, options) -} - -// GetService get a service -func (k *K3S) GetService(namespace string, name string) (*apiv1.Service, error) { - return k.CoreV1().Services(namespace).Get(name, metav1.GetOptions{}) -} diff --git a/deploy/k8s/kubernetes.go b/deploy/k8s/kubernetes.go index 8c7fe710..2975b16f 100644 --- a/deploy/k8s/kubernetes.go +++ b/deploy/k8s/kubernetes.go @@ -2,6 +2,7 @@ package k8s import ( "context" + "fmt" "os" "github.com/metrue/fx/deploy" @@ -62,14 +63,28 @@ func (k *K8S) Deploy( const replicas = int32(3) if _, err := k.GetDeployment(namespace, name); err != nil { // TODO enable passing replica from fx CLI - if _, err := k.CreateDeploymentWithInitContainer( - namespace, - name, - ports, - replicas, - selector, - ); err != nil { - return err + if os.Getenv("K3S") != "" { + // NOTE Doing docker build in initial container will fail when cluster is created by K3S + if _, err := k.CreateDeployment( + namespace, + name, + image, + ports, + replicas, + selector, + ); err != nil { + return err + } + } else { + if _, err := k.CreateDeploymentWithInitContainer( + namespace, + name, + ports, + replicas, + selector, + ); err != nil { + return err + } } } else { if _, err := k.UpdateDeployment( @@ -157,6 +172,19 @@ func (k *K8S) List(ctx context.Context, name string) ([]types.Service, error) { return []types.Service{}, nil } +// Ping health check of infra +func (k *K8S) Ping(ctx context.Context) error { + // Does not find any ping method for k8s + nodes, err := k.ListNodes() + if err != nil { + return err + } + if len(nodes.Items) <= 0 { + return fmt.Errorf("no available nodes") + } + return nil +} + var ( _ deploy.Deployer = &K8S{} ) diff --git a/deploy/k8s/node.go b/deploy/k8s/node.go new file mode 100644 index 00000000..82d8eac3 --- /dev/null +++ b/deploy/k8s/node.go @@ -0,0 +1,15 @@ +package k8s + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ListNodes list node +func (k *K8S) ListNodes() (*v1.NodeList, error) { + nodes, err := k.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + return nodes, nil +} diff --git a/deploy/mocks/deployer.go b/deploy/mocks/deployer.go index ff0cf2a6..0f08442e 100644 --- a/deploy/mocks/deployer.go +++ b/deploy/mocks/deployer.go @@ -105,3 +105,17 @@ func (mr *MockDeployerMockRecorder) List(ctx, name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockDeployer)(nil).List), ctx, name) } + +// Ping mocks base method +func (m *MockDeployer) Ping(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Ping", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Ping indicates an expected call of Ping +func (mr *MockDeployerMockRecorder) Ping(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ping", reflect.TypeOf((*MockDeployer)(nil).Ping), ctx) +} diff --git a/middlewares/setup.go b/middlewares/setup.go index 2232c2d1..8e539a24 100644 --- a/middlewares/setup.go +++ b/middlewares/setup.go @@ -10,19 +10,12 @@ import ( "github.com/metrue/fx/context" "github.com/metrue/fx/deploy" dockerDeployer "github.com/metrue/fx/deploy/docker" - k3sDeployer "github.com/metrue/fx/deploy/k3s" k8sDeployer "github.com/metrue/fx/deploy/k8s" - "github.com/metrue/fx/pkg/spinner" + "github.com/pkg/errors" ) // Setup create k8s or docker cli func Setup(ctx context.Contexter) (err error) { - const task = "setup" - spinner.Start(task) - defer func() { - spinner.Stop(task, err) - }() - fxConfig := ctx.Get("config").(*config.Config) cloud := fxConfig.Clouds[fxConfig.CurrentCloud] @@ -30,7 +23,7 @@ func Setup(ctx context.Contexter) (err error) { if cloud["type"] == config.CloudTypeDocker { docker, err := dockerHTTP.Create(cloud["host"], constants.AgentPort) if err != nil { - return err + return errors.Wrapf(err, "please make sure docker is installed and running on your host") } // TODO should clean up, but it needed in middlewares.Build ctx.Set("docker", docker) @@ -39,12 +32,7 @@ func Setup(ctx context.Contexter) (err error) { return err } } else if cloud["type"] == config.CloudTypeK8S { - if os.Getenv("K3S") != "" { - deployer, err = k3sDeployer.Create(cloud["kubeconfig"]) - if err != nil { - return err - } - } else if os.Getenv("KUBECONFIG") != "" { + if os.Getenv("KUBECONFIG") != "" { deployer, err = k8sDeployer.Create(cloud["kubeconfig"]) if err != nil { return err