From b81169253d7fd0c8e013cc9b8b4760efab0f5eaa Mon Sep 17 00:00:00 2001 From: Faiq Raza Date: Thu, 8 Mar 2018 16:43:24 -0800 Subject: [PATCH] VirtualBox/Vagrant Backend for CP (#815) * starts a vagrant cp interface * check the error * Start doing with vagrant * Add a brief doc explaining the virtual box provider * Spawn a node and get its address * Check in the dep * remove vbox tests so we can pass and more descriptive runner * implement the remove * pass in vagrant path as config * Remove the dep on uuid * remove false vagrant test --- poolmanager/server/cp/Readme.md | 31 ++++ poolmanager/server/cp/Vagrantfile | 76 ++++++++++ poolmanager/server/cp/vbox.go | 227 ++++++++++++++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 poolmanager/server/cp/Readme.md create mode 100644 poolmanager/server/cp/Vagrantfile create mode 100644 poolmanager/server/cp/vbox.go diff --git a/poolmanager/server/cp/Readme.md b/poolmanager/server/cp/Readme.md new file mode 100644 index 000000000..e78a34615 --- /dev/null +++ b/poolmanager/server/cp/Readme.md @@ -0,0 +1,31 @@ +#Vagrant testing + +We've created a control plane interface for local development using vagrant as a backend. It interacts with minikube, where the rest of the components of the fn project are expected to run for local development. + +##Getting it working + +In order to create virtual machines you're going to want to configure the minikube and the hosts provided to share a network adapter. If you haven't already, [download the binary](https://github.com/kubernetes/minikube) and run `minikube start --vm-provider=virtualbox`. This should configure a new virtual box host only network called `vboxnet0`. From there, you should be able to run thsis code as is to start VMs backed by virtual box. + +##Issues + +Occasionally, you may run into an issue with the DHCP server the virtual box configures and will not be able to start a server. + +If you see a message like this when running `vagrant up`: + +``` +A host only network interface you're attempting to configure via DHCP +already has a conflicting host only adapter with DHCP enabled. The +DHCP on this adapter is incompatible with the DHCP settings. Two +host only network interfaces are not allowed to overlap, and each +host only network interface can have only one DHCP server. Please +reconfigure your host only network or remove the virtual machine +using the other host only network. +``` + +Running the following command should clear all of these collision problems: + +`VBoxManage dhcpserver remove --netname HostInterfaceNetworking-vboxnet0` + +##Support + +This is only intended to be used to test distributed components any use further than this will not be supported. diff --git a/poolmanager/server/cp/Vagrantfile b/poolmanager/server/cp/Vagrantfile new file mode 100644 index 000000000..0e7c3c698 --- /dev/null +++ b/poolmanager/server/cp/Vagrantfile @@ -0,0 +1,76 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "terrywang/oraclelinux-7-x86_64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + # + config.vm.provider "virtualbox" do |vb| + config.vm.network "private_network", :type => 'dhcp', :name => 'vboxnet0', :adapter => 2 + end + # + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + + + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + # vb.memory = "1024" + # end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # apt-get update + # apt-get install -y apache2 + # SHELL +end diff --git a/poolmanager/server/cp/vbox.go b/poolmanager/server/cp/vbox.go new file mode 100644 index 000000000..e93fece0b --- /dev/null +++ b/poolmanager/server/cp/vbox.go @@ -0,0 +1,227 @@ +package cp + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + + idgen "github.com/fnproject/fn/api/id" +) + +const vboxNamePrefix = "fn-vagrant" + +var whichVBox *exec.Cmd + +func init() { + whichVBox = exec.Command("which", "vagrant") +} + +type VirtualBoxCP struct { + runnerMap map[string][]*Runner + vagrantPath string +} + +func NewVirtualBoxCP(vagrantPath string) (*VirtualBoxCP, error) { + runnerMap := make(map[string][]*Runner) + if err := whichVBox.Run(); err != nil { + return nil, err + } + return &VirtualBoxCP{ + runnerMap: runnerMap, + vagrantPath: vagrantPath, + }, nil +} + +func (v *VirtualBoxCP) provision() (*Runner, error) { + //set up dir + wd, err := os.Getwd() + if err != nil { + return nil, err + } + defer func() { + os.Chdir(wd) + }() + + node := newNodeName() + nodeDir, err := ioutil.TempDir(wd, node) + if err != nil { + return nil, err + } + //copy vagrant file into there + newVagrantFile := fmt.Sprintf("%s/%s", nodeDir, "Vagrantfile") + err = copyFile(v.vagrantPath, newVagrantFile) + if err != nil { + return nil, err + } + + err = os.Chdir(nodeDir) + if err != nil { + log.Println(err.Error()) + return nil, err + } + vboxProvision := exec.Command("vagrant", "up") + err = vboxProvision.Run() + if err != nil { + log.Println(err.Error()) + return nil, err + } + //Get the broadcast addr and call it a day + return getRunner(node) +} + +//Gets the address that its broadcasting at +//VBoxManage guestproperty get "cp_default_1520116902053_77841" "/VirtualBox/GuestInfo/Net/1/V4/Broadcast" +func getRunner(node string) (*Runner, error) { + //TODO make the vagrant file templated + vmsCmd := exec.Command("VBoxManage", "list", "vms") + var vmsOut bytes.Buffer + vmsCmd.Stdout = &vmsOut + err := vmsCmd.Run() + if err != nil { + return nil, err + } + vms := strings.Split(vmsOut.String(), "\n") + var realNode string + for _, candidate := range vms { + if strings.Contains(candidate, node) { + spl := strings.Split(candidate, " ") + realNode = spl[0] + } + } + //strip the quotes + if strings.Contains(realNode, "\"") { + realNode = realNode[1 : len(realNode)-1] + } + + //guestproperty get "fn-vagrant-6ae28c23-445e-4b0b-a2cf-0102e66ec57a766389779_default_1520288274551_74039" /VirtualBox/GuestInfo/Net/1/V4/Broadcast + args := []string{"guestproperty", "get", realNode, "/VirtualBox/GuestInfo/Net/1/V4/Broadcast"} + broadCastAddrCmd := exec.Command("VBoxManage", args...) + var out bytes.Buffer + broadCastAddrCmd.Stdout = &out + + var stdErr bytes.Buffer + broadCastAddrCmd.Stderr = &stdErr + + err = broadCastAddrCmd.Run() + if err != nil { + log.Println("error running", err.Error(), stdErr.String()) + return nil, err + } + addr := strings.Split(out.String(), ":") + if len(addr) != 2 { + return nil, fmt.Errorf("Unable to get address got:'%s' as output", out.String()) + } + return &Runner{ + Id: realNode, + Address: addr[1], + }, nil +} + +func (v *VirtualBoxCP) GetLBGRunners(lgbID string) ([]*Runner, error) { + runners, ok := v.runnerMap[lgbID] + if !ok { + return nil, errors.New("Not Found") + } + return runners, nil +} + +func (v *VirtualBoxCP) ProvisionRunners(lgbID string, n int) (int, error) { + runners := make([]*Runner, 0, n) + for i := 0; i < n; i++ { + runner, err := v.provision() + runners = append(runners, runner) + if err != nil { + return 0, err + } + } + v.runnerMap[lgbID] = runners + return n, nil +} + +func (v *VirtualBoxCP) RemoveRunner(lbgID string, id string) error { + wd, err := os.Getwd() + if err != nil { + return err + } + defer func() { + os.Chdir(wd) + }() + + runners, ok := v.runnerMap[lbgID] + if !ok { + return errors.New("No lgbID with this name") + } + //look for it in the customers map + found := false + for _, r := range runners { + if id == r.Id { + found = true + break + } + } + if found == false { + return errors.New("No VM by this ID") + } + //switch to the dir and remove it + //vm name is fn-vagrant-7183faa4-7321-47e9-8fd9-4a0aa1ac818e497509110_default_1520299457972_92567 everything before the first _ + split := strings.Split(id, "_") + dirName := split[0] + err = os.Chdir(dirName) + if err != nil { + log.Println(err.Error()) + return err + } + destroyCmd := exec.Command("vagrant", "destroy", "-f") + err = destroyCmd.Run() + if err != nil { + log.Println(err.Error()) + return err + } + // back to working dir and rm -rf ignore these erro + err = os.Chdir(wd) + if err != nil { + return err + } + err = os.RemoveAll(dirName) + if err != nil { + return err + } + return nil +} + +func newNodeName() string { + id := idgen.New() + return fmt.Sprintf("%s-%s", vboxNamePrefix, id.String()) +} + +//TODO move to a util folder if needed again +func copyFile(src string, dst string) error { + // Open the source file for reading + s, err := os.Open(src) + if err != nil { + return err + } + defer s.Close() + + // Open the destination file for writing + d, err := os.Create(dst) + if err != nil { + return err + } + + // Copy the contents of the source file into the destination file + if _, err := io.Copy(d, s); err != nil { + d.Close() + return err + } + + // Return any errors that result from closing the destination file + // Will return nil if no errors occurred + return d.Close() +}