feat: add cron prune and refactor api

.gitignore

@@ -1 +1,6 @@
-.DS_Store
+.DS_Store
+__pycache__/
+*.py[cod]
+*$py.class
+
+manifest.json

api/createContainer.py

@@ -1,47 +0,0 @@
-import docker
-
-
-class CreateContainer(object):
-    def __init__(self):
-
-        self.client = docker.from_env()
-
-    def is_create_container(self,
-                            mem,
-                            cpu,
-                            web_shell_port,
-                            container_name,
-                            os_name,
-                            open_port=None,
-                            rand_port=None) -> bool:
-
-        if open_port is None:
-            port_dict = {'1588/tcp': str(web_shell_port)}
-        else:
-            port_dict = {
-                '1588/tcp': str(web_shell_port),
-                str(open_port) + '/tcp': str(rand_port)
-            }
-
-        try:
-            self.client.containers.run(
-                image=os_name,
-                cpu_period=100000,
-                cpu_quota=int("%s0000" % cpu),
-                mem_limit="%sm" % mem,
-                name=container_name,
-                ports=port_dict,
-                restart_policy={"Name": "always"},
-                tty=True,
-                detach=True)
-        except Exception:
-            return False
-
-        else:
-            return True
-
-
-if __name__ == '__main__':
-    test = CreateContainer()
-    test.is_create_container("512", 1, 32233, "test_container",
-                             "instantbox/ubuntu:latest")

api/instantboxManager.py

@@ -0,0 +1,106 @@
+import docker
+import random
+import string
+import time
+import json
+
+
+class InstantboxManager(object):
+    CONTAINER_PREFIX = 'instantbox_managed_'
+    TIMEOUT_LABEL = 'org.instantbox.variables.EXPIRATION_TIMESTAMP'
+    OS_LIST = None
+
+    def __init__(self):
+        self.client = docker.from_env()
+
+        try:
+            with open('manifest.json', 'r') as os_manifest:
+                self.OS_LIST = json.load(os_manifest)
+        except Exception:
+            pass
+
+        if self.OS_LIST is None:
+            raise Exception(
+                'Could not load manifest.json. ' +
+                'Download it from https://get.instantbox.org/manifest.json'
+            )
+
+        self.AVAILABLE_OS_LIST = []
+        for os in self.OS_LIST:
+            for ver in os['subList']:
+                self.AVAILABLE_OS_LIST.append(ver['osCode'])
+
+    def is_create_container(self,
+                            mem,
+                            cpu,
+                            os_name,
+                            os_timeout,
+                            open_port=None):
+        if open_port is None:
+            port_dict = {'1588/tcp': None}
+        else:
+            port_dict = {'1588/tcp': None, '{}/tcp'.format(open_port): None}
+
+        container_name = self.generateContainerName()
+        try:
+            self.client.containers.run(
+                image=os_name,
+                cpu_period=100000,
+                cpu_quota=int('%s0000' % cpu),
+                mem_limit='%sm' % mem,
+                name=container_name,
+                ports=port_dict,
+                restart_policy={'Name': 'always'},
+                labels={self.TIMEOUT_LABEL: str.format('{:.0f}', os_timeout)},
+                tty=True,
+                detach=True,
+            )
+        except Exception:
+            return None
+        else:
+            return container_name
+
+    def get_container_ports(self, container_name):
+        try:
+            ports = self.client.containers.get(
+                container_name).attrs['NetworkSettings']['Ports']
+            return {
+                port: mapped_ports[0]['HostPort']
+                for port, mapped_ports in ports.items()
+            }
+        except Exception:
+            return None
+
+    def remove_timeout_containers(self):
+        for container in self.client.containers.list():
+            if container.name.startswith(self.CONTAINER_PREFIX):
+                timeout = container.labels.get(self.TIMEOUT_LABEL)
+                if timeout is not None and float(timeout) < time.time():
+                    self.is_rm_container(container.name)
+
+    def is_rm_container(self, container_id) -> bool:
+        try:
+            container = self.client.containers.get(container_id)
+        except docker.errors.NotFound:
+            return True
+        else:
+            if container.name.startswith(self.CONTAINER_PREFIX):
+                container.remove(force=True)
+            return True
+
+    def is_os_available(self, osCode=None) -> bool:
+        return osCode is not None and osCode in self.AVAILABLE_OS_LIST
+
+    def generateContainerName(self) -> str:
+        return self.CONTAINER_PREFIX + ''.join(
+            random.sample(string.ascii_letters + string.digits, 16))
+
+
+if __name__ == '__main__':
+    test = InstantboxManager()
+    container_name = test.is_create_container('512', 1,
+                                              'instantbox/ubuntu:latest',
+                                              time.time())
+    test.get_container_ports(container_name)
+    test.remove_timeout_containers()
+    test.is_rm_container(container_name)

api/killContainer.py

@@ -1,65 +0,0 @@
-import re
-import datetime
-import time
-import subprocess
-
-TIMEFORMAT = '%H:%M:%S'
-DAYEFORMAT = '%Y-%m-%d'
-
-
-def kill_container(containerId):
-    try:
-        print("[+] kill container is {}".format(containerId))
-        # subprocess.check_output("docker rm -f {}".format(containerId), shell=True)
-    except Exception:
-        return False
-    else:
-        return True
-
-
-def check_time(_day, _time, _containerId):
-
-    time_now = datetime.datetime.now().strftime(TIMEFORMAT)
-    day_now = datetime.datetime.now().strftime(DAYEFORMAT)
-
-    if day_now.split('-')[2] < _day.split('-')[2]:
-        raise RuntimeError
-    elif day_now.split('-')[2] == _day.split('-')[2]:
-
-        interval = int(time_now.split(':')[0]) - int(_time.split(':')[0])
-        if interval >= 24:
-            kill_container(_containerId)
-    else:
-        interval_day = int(day_now.split('-')[2]) - int(_day.split('-')[2])
-        if interval_day == 1:
-            interval_time = int(time_now.split(':')[0]) - int(
-                _time.split(':')[0])
-            if interval_time > 0:
-                kill_container(_containerId)
-        else:
-            kill_container(_containerId)
-
-
-def regex_time(status_list):
-    for item in status_list:
-        if item:
-            _day, _time, _containerId = re.findall(
-                r"(\d{4}-\d{2}-\d{1,2}) (\d{1,2}:\d{1,2}:\d{1,2}) \+\d{4} \w{3}\s+([\s\S]+)",
-                item)[0]
-            print(
-                "[+] check container({}) create time...".format(_containerId))
-            check_time(_day, _time, _containerId)
-            time.sleep(1)
-
-
-def get_status():
-    status = subprocess.check_output(
-        "docker ps --format \"{{.CreatedAt}}\t{{.ID}}\"", shell=True)
-    status_list = (status.decode("utf-8").strip()).split('\n')
-
-    return status_list
-
-
-if __name__ == '__main__':
-    status_list = get_status()
-    regex_time(status_list)

api/redisCli.py

@@ -1,49 +0,0 @@
-import redis
-
-
-class ConnectRedis(object):
-    def __init__(self):
-
-        redisPool = redis.ConnectionPool(host="redis", port=6379, db=0)
-        self.redisCli = redis.StrictRedis(connection_pool=redisPool)
-
-    def set_value(self, key: str, value: str) -> bool:
-
-        if key is not None or key != '':
-
-            self.redisCli.set(key, value)
-
-            return True
-        else:
-            return False
-
-    def get_value(self, key: str) -> str:
-
-        result = self.redisCli.get(key)
-        if result is not None:
-            return result.decode()
-        else:
-            return None
-
-    def is_container(self, containerId: str) -> bool:
-
-        result = self.redisCli.get(containerId)
-        if result is not None:
-            return True
-        else:
-            return False
-
-    def set_container(self, containerId: str) -> bool:
-
-        result = self.redisCli.set(containerId, containerId)
-        if result is not None:
-            return True
-        else:
-            return False
-
-
-if __name__ == '__main__':
-
-    test = ConnectRedis()
-    test.set_value("name", "Yuefeng Zhu")
-    print(test.get_value("name"))

api/rmContainer.py

@@ -1,22 +0,0 @@
-import docker
-
-
-class RmContainer(object):
-    def __init__(self):
-
-        self.client = docker.from_env()
-
-    def is_rm_container(self, container_id) -> bool:
-        try:
-            container = self.client.containers.get(container_id)
-        except docker.errors.NotFound:
-            return False
-        else:
-            container.remove(force=True)
-            return True
-
-
-if __name__ == '__main__':
-
-    test = RmContainer()
-    print(test.is_rm_container("redisInstance"))

docker-compose.yml

@@ -4,16 +4,12 @@ services:
   server:
     image: instantbox/instantbox:latest
     container_name: instantbox_server
-    links:
-      - redis
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       - SERVERURL=
-
-  redis:
-    image: redis:latest
-    container_name: instantbox_redis
+    networks:
+      - instantbox_net
 
   frontend:
     image: instantbox/instantbox-frontend:latest
@@ -22,3 +18,18 @@ services:
       - server
     ports:
       - 8888:80
+    networks:
+      - instantbox_net
+
+  cron:
+    image: xordiv/docker-alpine-cron
+    container_name: instantbox_cron
+    links:
+      - frontend
+    environment:
+      - CRON_STRINGS=* * * * * wget -qO /dev/null http://frontend/api/v2/superinspire/prune
+    networks:
+      - instantbox_net
+
+networks:
+  instantbox_net:

inspire.py

@@ -2,62 +2,33 @@
 
 import re
 import os
-import sys
 import json
 import time
-import random
-import string
-import subprocess
 from flask_cors import CORS
-from api.redisCli import ConnectRedis
-from api.rmContainer import RmContainer
 from flask import render_template, redirect
-from api.createContainer import CreateContainer
 from flask import Flask, request, Response, jsonify
+from api.instantboxManager import InstantboxManager
 
 app = Flask(__name__)
 CORS(app, resources=r'/*')
 
-redisCli = ConnectRedis()
-create_container_client = CreateContainer()
-rm_container_client = RmContainer()
+instantboxManager = InstantboxManager()
 
 SERVERURL = os.environ.get('SERVERURL')
-
-OS_LIST = None
-with open("manifest.json", "r") as os_manifest:
-    OS_LIST = json.load(os_manifest)
-if OS_LIST is None:
-    raise Exception('Could not load manifest.json')
-
-AVAILABLE_OS = []
-for os in OS_LIST:
-    for ver in os['subList']:
-        AVAILABLE_OS.append(ver['osCode'])
+if SERVERURL is None:
+    SERVERURL = ''
 
 
-def randPort():
-    rand_port = random.randint(1, 65536)
-    if (6000 <= rand_port <= 7000) or (rand_port == 22):
-        randPort()
-    else:
-        try:
-            subprocess.check_output("lsof -i:%s" % (rand_port), shell=True)
-        except subprocess.CalledProcessError:
-            return rand_port
-        else:
-            randPort()
-
-
-def genString():
-    salt = ''.join(random.sample(string.ascii_letters + string.digits, 16))
-    return salt
+@app.route('/v2/superinspire')
+def hello():
+    return 'hello'
 
 
 @app.route('/v2/superinspire/getOSList')
 def returnList():
 
-    response = Response(json.dumps(OS_LIST), mimetype='application/json')
+    response = Response(
+        json.dumps(instantboxManager.OS_LIST), mimetype='application/json')
 
     response.headers.add('Server', 'python flask')
     response.headers['Access-Control-Allow-Origin'] = '*'
@@ -66,53 +37,39 @@ def returnList():
     return response
 
 
-@app.route('/v2/superinspire')
-def hello():
-    return 'hello'
-
-
 @app.route('/v2/superinspire/rmOS')
 def rmOS():
     try:
-        containerId = request.args.get("containerId")
-        timestamp = request.args.get("timestamp")
-        shareUrl = request.args.get("shareUrl")
+        containerId = request.args.get('containerId')
+        timestamp = request.args.get('timestamp')
+        shareUrl = request.args.get('shareUrl')
     except Exception:
         response = Response(
             json.dumps({
-                "message": "Arguments ERROR",
-                "statusCode": 0
+                'message': 'Arguments ERROR',
+                'statusCode': 0
             }),
             mimetype='application/json')
     else:
-        if redisCli.is_container(containerId):
-            try:
-                isSuccess = rm_container_client.is_rm_container(containerId)
-                if not isSuccess:
-                    raise Exception
+        try:
+            isSuccess = instantboxManager.is_rm_container(containerId)
+            if not isSuccess:
+                raise Exception
 
-            except Exception:
-                response = Response(
-                    json.dumps({
-                        "message": "RM docker containers ERROR",
-                        "shareUrl": "",
-                        "statusCode": 0,
-                    }),
-                    mimetype='application/json')
-            else:
-                response = Response(
-                    json.dumps({
-                        "message": "SUCCESS",
-                        "statusCode": 1,
-                        "containerId": containerId,
-                    }),
-                    mimetype='application/json')
+        except Exception:
+            response = Response(
+                json.dumps({
+                    'message': 'RM docker containers ERROR',
+                    'shareUrl': '',
+                    'statusCode': 0,
+                }),
+                mimetype='application/json')
         else:
             response = Response(
                 json.dumps({
-                    "message": "docker containers not exist ERROR",
-                    "statusCode": 0,
-                    "containerId": containerId,
+                    'message': 'SUCCESS',
+                    'statusCode': 1,
+                    'containerId': containerId,
                 }),
                 mimetype='application/json')
 
@@ -125,93 +82,100 @@ def rmOS():
 
 @app.route('/v2/superinspire/getOS')
 def getOS():
-
-    shareUrl = "http://{0}:{1}"
-    openPort = ''
+    open_port = None
 
     try:
-        os_name = request.args.get("os")
+        os_name = request.args.get('os')
+        if not instantboxManager.is_os_available(os_name):
+            raise Exception
     except Exception:
         response = Response(
             json.dumps({
-                "message": "OS Arguments ERROR",
-                "statusCode": 0
+                'message': 'The image is not supported at this time ERROR',
+                'statusCode': 0
             }),
             mimetype='application/json')
     else:
-        try:
-            os_mem = request.args.get("mem")
-            os_cpu = request.args.get("cpu")
-            os_port = request.args.get("port")
-            os_timeout = request.args.get("timeout")
-        except Exception:
-            if os_name not in AVAILABLE_OS:
-                response = Response(
-                    json.dumps({
-                        "message":
-                        "The image is not supported at this time ERROR",
-                        "statusCode": 0
-                    }),
-                    mimetype='application/json')
+        os_mem = request.args.get('mem')
+        os_cpu = request.args.get('cpu')
+        os_port = request.args.get('port')
+        os_timeout = request.args.get('timeout')
+
+        if os_mem is None:
+            os_mem = 512
+        if os_cpu is None:
+            os_cpu = 1
+        max_timeout = 3600 * 24 + time.time()
+        if os_timeout is None:
+            os_timeout = max_timeout
         else:
+            os_timeout = min(float(os_timeout), max_timeout)
 
-            if os_mem is None:
-                os_mem = 512
-            if os_cpu is None:
-                os_cpu = 1
-            if os_timeout:
-                os_timeout = 3600 * 24 + time.time()
+        try:
+            container_name = instantboxManager.is_create_container(
+                mem=os_mem,
+                cpu=os_cpu,
+                os_name=os_name,
+                open_port=os_port,
+                os_timeout=os_timeout,
+            )
 
-            rand_string = genString()
-            webShellPort = randPort()
-            try:
-                if os_port is None:
-                    isSuccess = create_container_client.is_create_container(
-                        mem=os_mem,
-                        cpu=os_cpu,
-                        web_shell_port=webShellPort,
-                        container_name=rand_string,
-                        os_name=os_name,
-                    )
-
-                else:
-                    openPort = randPort()
-                    isSuccess = create_container_client.is_create_container(
-                        mem=os_mem,
-                        cpu=os_cpu,
-                        web_shell_port=webShellPort,
-                        container_name=rand_string,
-                        os_name=os_name,
-                        open_port=os_port,
-                        rand_port=openPort)
-
-                if not isSuccess:
-                    raise Exception
-            except Exception:
-                response = Response(
-                    json.dumps({
-                        "message": "RUN docker containers ERROR",
-                        "shareUrl": "",
-                        "statusCode": 0,
-                    }),
-                    mimetype='application/json')
+            if container_name is None:
+                raise Exception
             else:
-                redisCli.set_container(rand_string)
+                ports = instantboxManager.get_container_ports(container_name)
+                webshell_port = ports['1588/tcp']
+                if os_port is not None:
+                    open_port = ports['{}/tcp'.format(os_port)]
 
-                response = Response(
-                    json.dumps({
-                        "message":
-                        "SUCCESS",
-                        "shareUrl":
-                        shareUrl.format(SERVERURL, webShellPort),
-                        "openPort":
-                        openPort,
-                        "statusCode":
-                        1,
-                        "containerId":
-                        rand_string,
-                    }),
-                    mimetype='application/json')
+        except Exception:
+            response = Response(
+                json.dumps({
+                    'message': 'RUN docker containers ERROR',
+                    'shareUrl': '',
+                    'statusCode': 0,
+                }),
+                mimetype='application/json')
+        else:
+            response = Response(
+                json.dumps({
+                    'message':
+                    'SUCCESS',
+                    'shareUrl':
+                    'http://{}:{}'.format(SERVERURL, webshell_port),
+                    'openPort':
+                    open_port,
+                    'statusCode':
+                    1,
+                    'containerId':
+                    container_name,
+                }),
+                mimetype='application/json')
+
+    response.headers.add('Server', 'python flask')
+    response.headers['Access-Control-Allow-Origin'] = '*'
+    response.headers['Access-Control-Allow-Methods'] = 'GET,POST'
+    response.headers['Access-Control-Allow-Headers'] = 'x-requested-with'
+    return response
+
+
+@app.route('/v2/superinspire/prune')
+def pruneTimedoutOS():
+    try:
+        instantboxManager.remove_timeout_containers()
+        response = Response(
+            json.dumps({
+                'message': 'Success',
+                'statusCode': 1
+            }),
+            mimetype='application/json')
+    except Exception:
+        response = Response(
+            json.dumps({
+                'message': 'ERROR',
+                'statusCode': 0
+            }),
+            mimetype='application/json')
 
     response.headers.add('Server', 'python flask')
     response.headers['Access-Control-Allow-Origin'] = '*'
@@ -222,4 +186,4 @@ def getOS():
 
 if __name__ == '__main__':
 
-    app.run(host="0.0.0.0", port=int(65501), debug=False)
+    app.run(host='0.0.0.0', port=int(65501), debug=False)

requirement.txt

@@ -1,4 +1,3 @@
 flask==1.0.2
 flask_cors==3.0.6
-redis==2.10.6
-docker==3.6.0
+docker==3.6.0