Merge pull request #383 from FirmlyReality/master

Can migrate container at the backend

src/master/httprest.py

@@ -389,6 +389,49 @@ def save_cluster(user, beans, form):
     finally:
         G_ulockmgr.release(user)
 
+@app.route("/admin/migrate_cluster/", methods=['POST'])
+@auth_key_required
+def migrate_cluster():
+    global G_vclustermgr
+    global G_ulockmgr
+    user = request.form.get('username',None)
+    if user is None:
+        return json.dumps({'success':'false', 'message':'User is required!'})
+    clustername = request.form.get('clustername', None)
+    if (clustername == None):
+        return json.dumps({'success':'false', 'message':'clustername is null'})
+    new_hosts = request.form.get('new_hosts', None)
+    if (new_hosts == None):
+        return json.dumps({'success':'false', 'message':'new_hosts is null'})
+    new_host_list = new_hosts.split(',')
+    G_ulockmgr.acquire(user)
+    auth_key = env.getenv('AUTH_KEY')
+    try:
+        logger.info ("handle request : migrate cluster to %s. user:%s clustername:%s" % (str(new_hosts), user, clustername))
+        res = post_to_user("/master/user/groupinfo/", {'auth_key':auth_key})
+        groups = json.loads(res['groups'])
+        quotas = {}
+        for group in groups:
+            #logger.info(group)
+            quotas[group['name']] = group['quotas']
+        rc_info = post_to_user("/master/user/recoverinfo/", {'username':user,'auth_key':auth_key})
+        groupname = rc_info['groupname']
+        user_info = {"data":{"id":rc_info['uid'],"groupinfo":quotas[groupname]}}
+
+        logger.info("Migrate cluster for user(%s) cluster(%s) to new_hosts(%s). user_info(%s)"
+                    %(clustername, user, str(new_host_list), user_info))
+
+        [status,msg] = G_vclustermgr.migrate_cluster(clustername, user, new_host_list, user_info)
+        if not status:
+            logger.error(msg)
+            return json.dumps({'success':'false', 'message': msg})
+        return json.dumps({'success':'true', 'action':'migrate_container'})
+    except Exception as ex:
+        logger.error(traceback.format_exc())
+        return json.dumps({'success':'false', 'message': str(ex)})
+    finally:
+        G_ulockmgr.release(user)
+
 
 @app.route("/image/list/", methods=['POST'])
 @login_required

src/master/userManager.py

@@ -411,6 +411,7 @@ class userManager:
                 "success":'true',
                 "data":{
                     "username" : user.username,
+                    "id": user.id,
                     "password" : user.password,
                     "avatar" : user.avatar,
                     "nickname" : user.nickname,
@@ -440,6 +441,7 @@ class userManager:
             "success": 'true',
             "data":{
                 "username" : user.username,
+                "id": user.id,
                 "password" : user.password,
                 "avatar" : user.avatar,
                 "nickname" : user.nickname,

src/master/vclustermgr.py

@@ -728,6 +728,94 @@ class VclusterMgr(object):
             full_clusters.append(single_cluster)'''
         return [True, clusters]
 
+    def migrate_container(self, clustername, username, containername, new_host, user_info):
+        [status, info] = self.get_clusterinfo(clustername, username)
+        if not status:
+            return [False, "cluster not found"]
+        if info['status'] != 'stopped':
+            return [False, 'cluster is not stopped']
+
+        con_db = Container.query.get(containername)
+        if con_db is None:
+            return [False, 'Container not found']
+        if con_db.host == new_host:
+            return [False, 'Container has been on the new host']
+
+        oldworker = self.nodemgr.ip_to_rpc(con_db.host)
+        if oldworker is None:
+            return [False, "Old host worker can't be found or has been stopped."]
+        oldworker.stop_container(containername)
+        imagename = "migrate-" + containername + "-" + datetime.datetime.now().strftime("%Y-%m-%d")
+        logger.info("Save Image for container:%s imagename:%s host:%s"%(containername, imagename, con_db.host))
+        status,msg = oldworker.create_image(username,imagename,containername,"",10000)
+        if not status:
+            return [False, msg]
+        #con_db.lastsave = datetime.datetime.now()
+        #con_db.image = imagename
+
+        self.networkmgr.load_usrgw(username)
+        proxy_server_ip = self.networkmgr.usrgws[username]
+        [status, proxy_public_ip] = self.etcd.getkey("machines/publicIP/"+proxy_server_ip)
+        if not status:
+            self.imgmgr.removeImage(username,imagename)
+            logger.error("Fail to get proxy_public_ip %s."%(proxy_server_ip))
+            return [False, "Fail to get proxy server public IP."]
+        uid = user_info['data']['id']
+        setting = {
+                'cpu': con_db.setting_cpu,
+                'memory': con_db.setting_mem,
+                'disk': con_db.setting_disk
+                }
+        _, clusterid, cid = containername.split('-')
+        hostname = "host-"+str(cid)
+        gateway = self.networkmgr.get_usergw(username)
+        image = {'name':imagename,'type':'private','owner':username }
+        logger.info("Migrate: proxy_ip:%s uid:%s setting:%s clusterid:%s cid:%s hostname:%s gateway:%s image:%s"
+                    %(proxy_public_ip, str(uid), str(setting), clusterid, cid, hostname, gateway, str(image)))
+        logger.info("Migrate: create container(%s) on new host %s"%(containername, new_host))
+
+        worker = self.nodemgr.ip_to_rpc(new_host)
+        if worker is None:
+            self.imgmgr.removeImage(username,imagename)
+            return [False, "New host worker can't be found or has been stopped."]
+        status,msg = worker.create_container(containername, proxy_public_ip, username, uid, json.dumps(setting),
+                     clustername, str(clusterid), str(cid), hostname, con_db.ip, gateway, json.dumps(image))
+        if not status:
+            self.imgmgr.removeImage(username,imagename)
+            return [False, msg]
+        con_db.host = new_host
+        db.session.commit()
+        oldworker.delete_container(containername)
+        self.imgmgr.removeImage(username,imagename)
+        return [True,""]
+
+    def migrate_cluster(self, clustername, username, new_host_list, user_info):
+        [status, info] = self.get_clusterinfo(clustername, username)
+        if not status:
+            return [False, "cluster not found"]
+        prestatus = info['status']
+        self.stop_cluster(clustername, username)
+        for container in info['containers']:
+            if container['host'] in new_host_list:
+                continue
+            random.shuffle(new_host_list)
+            for new_host in new_host_list:
+                status,msg = self.migrate_container(clustername,username,container['containername'],new_host,user_info)
+                if status:
+                    break
+                else:
+                    logger.error(msg)
+            else:
+                if prestatus == 'running':
+                    self.start_cluster(clustername, username, user_info)
+                return [False, msg]
+        logger.info("[Migrate] prestatus:%s for cluster(%s) user(%s)"%(prestatus, clustername, username))
+        if prestatus == 'running':
+            status, msg = self.start_cluster(clustername, username, user_info)
+            if not status:
+                return [False, msg]
+        return [True, ""]
+
     def is_cluster(self, clustername, username):
         [status, clusters] = self.list_clusters(username)
         if clustername in clusters: