From 1148f766e50db179a3f7f4a250467f29d748ae82 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 9 May 2017 01:06:39 +0800 Subject: [PATCH 01/16] add vxlan support --- conf/container.conf | 14 +++++++------- conf/lxc-script/lxc-ifdown | 6 +++++- conf/lxc-script/lxc-ifup | 8 ++++++-- src/container.py | 13 ++++++++++--- src/httprest.py | 10 +++++++--- src/nettools.py | 28 +++++++++++++++++++--------- src/network.py | 20 +++++++++++++++++--- src/userManager.py | 5 +++-- src/vclustermgr.py | 24 +++++++++++++++--------- src/worker.py | 1 + user/user.py | 9 +++++++++ 11 files changed, 99 insertions(+), 39 deletions(-) diff --git a/conf/container.conf b/conf/container.conf index bf7f1be..1f3db65 100644 --- a/conf/container.conf +++ b/conf/container.conf @@ -1,8 +1,8 @@ -# This is the common container.conf for all containers. +# This is the common container.conf for all containers. # If want set custom settings, you have two choices: # 1. Directly modify this file, which is not recommend, because the # setting will be overriden when new version container.conf released. -# 2. Use a custom config file in this conf directory: lxc.custom.conf, +# 2. Use a custom config file in this conf directory: lxc.custom.conf, # it uses the same grammer as container.conf, and will be merged # with the default container.conf by docklet at runtime. # @@ -15,7 +15,7 @@ lxc.include = /usr/share/lxc/config/ubuntu.common.conf ############## DOCKLET CONFIG ############## -# Setup 0 tty devices +# Setup 0 tty devices lxc.tty = 0 lxc.rootfs = %ROOTFS% @@ -25,8 +25,8 @@ lxc.network.type = veth lxc.network.name = eth0 # veth.pair is limited in 16 bytes lxc.network.veth.pair = %VETHPAIR% -lxc.network.script.up = Bridge=docklet-br VLANID=%VLANID% %LXCSCRIPT%/lxc-ifup -lxc.network.script.down = Bridge=docklet-br %LXCSCRIPT%/lxc-ifdown +lxc.network.script.up = Bridge=docklet-br-%UserID% GatewayHost=%GatewayHost% UserID=%UserID% %LXCSCRIPT%/lxc-ifup +lxc.network.script.down = Bridge=docklet-br-%UserID% %LXCSCRIPT%/lxc-ifdown lxc.network.ipv4 = %IP% lxc.network.ipv4.gateway = %GATEWAY% lxc.network.flags = up @@ -50,7 +50,7 @@ lxc.mount.entry = %FS_PREFIX%/global/users/%USERNAME%/ssh %ROOTFS%/root/.ssh non lxc.mount.entry = %FS_PREFIX%/local/temp/%LXCNAME%/ %ROOTFS%/tmp none bind,rw,create=dir 0 0 # setting hostname -lxc.hook.pre-start = HNAME=%HOSTNAME% %LXCSCRIPT%/lxc-prestart +lxc.hook.pre-start = HNAME=%HOSTNAME% %LXCSCRIPT%/lxc-prestart # setting nfs softlink -#lxc.hook.mount = %LXCSCRIPT%/lxc-mount +#lxc.hook.mount = %LXCSCRIPT%/lxc-mount diff --git a/conf/lxc-script/lxc-ifdown b/conf/lxc-script/lxc-ifdown index 50d05b3..b0324bf 100755 --- a/conf/lxc-script/lxc-ifdown +++ b/conf/lxc-script/lxc-ifdown @@ -4,6 +4,10 @@ # $2 : net # $3 : network flags, up or down # $4 : network type, for example, veth -# $5 : value of lxc.network.veth.pair +# $5 : value of lxc.network.veth.pair ovs-vsctl --if-exists del-port $Bridge $5 +cnt=$(ovs-vsctl list-ports ${Bridge} | wc -l) +if [ "$cnt" = "1" ]; then + ovs-vsctl del-br $Bridge +fi diff --git a/conf/lxc-script/lxc-ifup b/conf/lxc-script/lxc-ifup index 0bf93b7..5bcf1a8 100755 --- a/conf/lxc-script/lxc-ifup +++ b/conf/lxc-script/lxc-ifup @@ -5,6 +5,10 @@ # $2 : net # $3 : network flags, up or down # $4 : network type, for example, veth -# $5 : value of lxc.network.veth.pair +# $5 : value of lxc.network.veth.pair -ovs-vsctl --may-exist add-port $Bridge $5 tag=$VLANID +ovs-vsctl --may-exist add-br $Bridge +if [ "$GatewayIP" != "NULL" ]; then + ovs-vsctl --may-exist add-port $Bridge vxlan-$UserID-$GatewayHost -- set interface vxlan-$UserID-$GatewayHost type=vxlan options:remote_ip=$GatewayHost options:key=$UserID +fi +ovs-vsctl --may-exist add-port $Bridge $5 diff --git a/src/container.py b/src/container.py index ae99f61..e5fc012 100755 --- a/src/container.py +++ b/src/container.py @@ -2,6 +2,7 @@ import subprocess, os, json import imagemgr +import network from log import logger import env from lvmtool import sys_run, check_volume @@ -23,7 +24,7 @@ class Container(object): self.imgmgr = imagemgr.ImageMgr() self.historymgr = History_Manager() - def create_container(self, lxc_name, proxy_server_ip, username, setting, clustername, clusterid, containerid, hostname, ip, gateway, vlanid, image): + def create_container(self, lxc_name, proxy_server_ip, username, uid, setting, clustername, clusterid, containerid, hostname, ip, gateway, image): logger.info("create container %s of %s for %s" %(lxc_name, clustername, username)) try: setting = json.loads(setting) @@ -37,7 +38,7 @@ class Container(object): #Ret = subprocess.run([self.libpath+"/lxc_control.sh", # "create", lxc_name, username, str(clusterid), hostname, - # ip, gateway, str(vlanid), str(cpu), str(memory)], stdout=subprocess.PIPE, + # ip, gateway, str(cpu), str(memory)], stdout=subprocess.PIPE, # stderr=subprocess.STDOUT,shell=False, check=True) rootfs = "/var/lib/lxc/%s/rootfs" % lxc_name @@ -60,7 +61,13 @@ class Container(object): content = content.replace("%CLUSTERID%",str(clusterid)) content = content.replace("%LXCSCRIPT%",env.getenv("LXC_SCRIPT")) content = content.replace("%LXCNAME%",lxc_name) - content = content.replace("%VLANID%",str(vlanid)) + content = content.replace("%UserID%",str(uid)) + net_dev = env.getenv("NETWORK_DEVICE") + ipaddr = network.getip(net_dev) + if ipaddr == proxy_server_ip: + content = content.replace("%GatewayHost%","NULL") + else: + content = content.replace("%GatewayHost%",proxy_server_ip) content = content.replace("%CLUSTERNAME%", clustername) content = content.replace("%VETHPAIR%", str(clusterid)+'-'+str(containerid)) return content diff --git a/src/httprest.py b/src/httprest.py index ce17063..5afad84 100755 --- a/src/httprest.py +++ b/src/httprest.py @@ -139,6 +139,7 @@ def create_cluster(user, beans, form): image['type'] = form.get("imagetype", None) image['owner'] = form.get("imageowner", None) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) + uid = user_info['id'] user_info = json.dumps(user_info) logger.info ("handle request : create cluster %s with image %s " % (clustername, image['name'])) setting = { @@ -151,7 +152,7 @@ def create_cluster(user, beans, form): result = res.get('result') if not status: return json.dumps({'success':'false', 'action':'create cluster', 'message':result}) - [status, result] = G_vclustermgr.create_cluster(clustername, user, image, user_info, setting) + [status, result] = G_vclustermgr.create_cluster(clustername, user, uid, image, user_info, setting) if status: return json.dumps({'success':'true', 'action':'create cluster', 'message':result}) else: @@ -173,6 +174,7 @@ def scaleout_cluster(user, beans, form): image['type'] = form.get("imagetype", None) image['owner'] = form.get("imageowner", None) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) + uid = user_info['id'] user_info = json.dumps(user_info) setting = { 'cpu': form.get('cpuSetting'), @@ -184,7 +186,7 @@ def scaleout_cluster(user, beans, form): result = res.get('result') if not status: return json.dumps({'success':'false', 'action':'scale out', 'message': result}) - [status, result] = G_vclustermgr.scale_out_cluster(clustername, user, image, user_info, setting) + [status, result] = G_vclustermgr.scale_out_cluster(clustername, user, uid, image, user_info, setting) if status: return json.dumps({'success':'true', 'action':'scale out', 'message':result}) else: @@ -215,8 +217,10 @@ def start_cluster(user, beans, form): clustername = form.get('clustername', None) if (clustername == None): return json.dumps({'success':'false', 'message':'clustername is null'}) + user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) + uid = user_info['id'] logger.info ("handle request : start cluster %s" % clustername) - [status, result] = G_vclustermgr.start_cluster(clustername, user) + [status, result] = G_vclustermgr.start_cluster(clustername, user, uid) if status: return json.dumps({'success':'true', 'action':'start cluster', 'message':result}) else: diff --git a/src/nettools.py b/src/nettools.py index e3cef28..151fb55 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import subprocess +import subprocess,network class ipcontrol(object): @staticmethod @@ -12,7 +12,7 @@ class ipcontrol(object): if len(line)==0: continue # Level 1 : first line of one link - if line[0] != ' ': + if line[0] != ' ': blocks = line.split() thislink = blocks[1].strip(':') links[thislink] = {} @@ -25,7 +25,7 @@ class ipcontrol(object): links[thislink]['inet'] = [] links[thislink]['inet'].append(blocks[1]) # we just need inet (IPv4) - else: + else: pass # Level 3 or more : no need for us else: @@ -40,7 +40,7 @@ class ipcontrol(object): return [True, list(links.keys())] except subprocess.CalledProcessError as suberror: return [False, "list links failed : %s" % suberror.stdout.decode('utf-8')] - + @staticmethod def link_exist(linkname): try: @@ -109,7 +109,7 @@ class ipcontrol(object): return [False, "delete address failed : %s" % suberror.stdout.decode('utf-8')] -# ovs-vsctl list-br +# ovs-vsctl list-br # ovs-vsctl br-exists # ovs-vsctl add-br # ovs-vsctl del-br @@ -153,7 +153,7 @@ class ovscontrol(object): @staticmethod def add_bridge(bridge): try: - subprocess.run(['ovs-vsctl', 'add-br', str(bridge)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True) + subprocess.run(['ovs-vsctl', '--may-exist', 'add-br', str(bridge)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True) return [True, str(bridge)] except subprocess.CalledProcessError as suberror: return [False, "add bridge failed : %s" % suberror.stdout.decode('utf-8')] @@ -206,6 +206,14 @@ class ovscontrol(object): except subprocess.CalledProcessError as suberror: return [False, "add port failed : %s" % suberror.stdout.decode('utf-8')] + @staticmethod + def add_port_vxlan(bridge, port, remote, key): + try: + subprocess.run(['ovs-vsctl', '--may-exist', 'add-port', str(bridge), str(port), '--', 'set', 'interface', str(port), 'type=vxlan', 'options:remote_ip='+str(remote), 'options:key='+str(key)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True) + return [True, str(port)] + except subprocess.CalledProcessError as suberror: + return [False, "add port failed : %s" % suberror.stdout.decode('utf-8')] + @staticmethod def set_port_tag(port, tag): try: @@ -256,7 +264,8 @@ class netcontrol(object): return ovscontrol.del_port(bridge, gwport) @staticmethod - def check_gw(bridge, gwport, addr, tag): + def check_gw(bridge, gwport, uid, addr, tag): + ovscontrol.add_bridge(bridge) if not netcontrol.gw_exists(bridge, gwport): return netcontrol.setup_gw(bridge, gwport, addr, tag) [status, info] = ipcontrol.link_info(gwport) @@ -272,5 +281,6 @@ class netcontrol(object): if info['state'] == 'DOWN': ipcontrol.up_link(gwport) return [True, "check gateway port %s" % gwport] - - + + #@staticmethod + #def check_usernet(uid, GatewayHost): diff --git a/src/network.py b/src/network.py index 5e6250a..350b4eb 100755 --- a/src/network.py +++ b/src/network.py @@ -524,7 +524,7 @@ class NetworkMgr(object): del self.users[username] return [True, 'delete user success'] - def check_usergw(self, username, nodemgr, distributedgw=False): + def check_usergw(self, username, uid, nodemgr, distributedgw=False): self.load_usrgw(username) if username not in self.usrgws.keys(): return [False, 'user does not exist.'] @@ -535,13 +535,27 @@ class NetworkMgr(object): self.del_usrgw(username,nodemgr) self.usrgws[username] = self.masterip self.dump_usrgw(username) - netcontrol.check_gw('docklet-br', username, self.users[username].get_gateway_cidr(), str(self.users[username].vlanid)) + netcontrol.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr(), str(self.users[username].vlanid)) else: worker = nodemgr.ip_to_rpc(ip) - worker.check_gw('docklet-br', username, self.users[username].get_gateway_cidr(), str(self.users[username].vlanid)) + worker.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr(), str(self.users[username].vlanid)) del self.users[username] return [True, 'check gw ok'] + def check_uservxlan(self, username, uid, remote, nodemgr, distributedgw=False): + self.load_usrgw(username) + if username not in self.usrgws.keys(): + return [False, 'user does not exist.'] + ip = self.usrgws[username] + if not distributedgw: + if not remote == self.masterip: + ovscontrol.add_port_vxlan('docklet-br-'+str(uid), 'vxlan-'+str(uid)+'-'+remote, remote, uid) + else: + if not remote == ip: + worker = nodemgr.ip_to_rpc(ip) + worker.add_port_vxlan('docklet-br-'+str(uid), 'vxlan-'+str(uid)+'-'+remote, remote, uid) + return [True, 'check vxlan ok'] + def has_user(self, username): [status, _value] = self.etcd.getkey("network/users/"+username) return status diff --git a/src/userManager.py b/src/userManager.py index d1d5b39..5b2e8a8 100755 --- a/src/userManager.py +++ b/src/userManager.py @@ -440,6 +440,7 @@ class userManager: "success": 'true', "data":{ "username" : user.username, + "id": user.id, "password" : user.password, "avatar" : user.avatar, "nickname" : user.nickname, @@ -1003,7 +1004,7 @@ class userManager: accountfile.write(json.dumps(account)) accountfile.close() return {"success": 'true'} - + @administration_required def cloud_account_del(*args, **kwargs): form = kwargs.get('form') @@ -1019,7 +1020,7 @@ class userManager: accountfile.write(json.dumps(account)) accountfile.close() return {"success": 'true'} - + @administration_required def cloud_account_modify(*args, **kwargs): form = kwargs.get('form') diff --git a/src/vclustermgr.py b/src/vclustermgr.py index e4416e0..63f85f5 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -4,6 +4,7 @@ import os, random, json, sys, imagemgr import datetime import xmlrpc.client +from httprest import post_to_user from log import logger import env import proxytool @@ -48,10 +49,12 @@ class VclusterMgr(object): def recover_allclusters(self): logger.info("recovering all vclusters for all users...") usersdir = self.fspath+"/global/users/" + auth_key = env.getenv('AUTH_KEY') for user in os.listdir(usersdir): for cluster in self.list_clusters(user)[1]: logger.info ("recovering cluster:%s for user:%s ..." % (cluster, user)) - self.recover_cluster(cluster, user) + res = post_to_user('/user/uid/',{'username':user,'auth_key':auth_key}) + self.recover_cluster(cluster, user, res['uid']) logger.info("recovered all vclusters for all users") def mount_allclusters(self): @@ -81,7 +84,7 @@ class VclusterMgr(object): self.detach_cluster(cluster, user) logger.info("detached all vclusters for all users") - def create_cluster(self, clustername, username, image, user_info, setting): + def create_cluster(self, clustername, username, uid, image, user_info, setting): if self.is_cluster(clustername, username): return [False, "cluster:%s already exists" % clustername] clustersize = int(self.defaultsize) @@ -127,7 +130,7 @@ class VclusterMgr(object): lxc_name = username + "-" + str(clusterid) + "-" + str(i) hostname = "host-"+str(i) logger.info ("create container with : name-%s, username-%s, clustername-%s, clusterid-%s, hostname-%s, ip-%s, gateway-%s, image-%s" % (lxc_name, username, clustername, str(clusterid), hostname, ips[i], gateway, image_json)) - [success,message] = oneworker.create_container(lxc_name, proxy_server_ip, username, json.dumps(setting) , clustername, str(clusterid), str(i), hostname, ips[i], gateway, str(vlanid), image_json) + [success,message] = oneworker.create_container(lxc_name, proxy_server_ip, username, uid, json.dumps(setting) , clustername, str(clusterid), str(i), hostname, ips[i], gateway, image_json) if success is False: logger.info("container create failed, so vcluster create failed") return [False, message] @@ -144,7 +147,7 @@ class VclusterMgr(object): clusterfile.close() return [True, info] - def scale_out_cluster(self,clustername,username,image,user_info, setting): + def scale_out_cluster(self,clustername,username,uid, image,user_info, setting): if not self.is_cluster(clustername,username): return [False, "cluster:%s not found" % clustername] workers = self.nodemgr.get_nodeips() @@ -169,7 +172,7 @@ class VclusterMgr(object): lxc_name = username + "-" + str(clusterid) + "-" + str(cid) hostname = "host-" + str(cid) proxy_server_ip = clusterinfo['proxy_server_ip'] - [success, message] = oneworker.create_container(lxc_name, username, json.dumps(setting), clustername, clusterid, str(cid), hostname, ip, gateway, str(vlanid), image_json) + [success, message] = oneworker.create_container(lxc_name, username, uid, json.dumps(setting), clustername, clusterid, str(cid), hostname, ip, gateway, image_json) if success is False: logger.info("create container failed, so scale out failed") return [False, message] @@ -357,7 +360,7 @@ class VclusterMgr(object): return [True, info] - def start_cluster(self, clustername, username): + def start_cluster(self, clustername, username, uid): [status, info] = self.get_clusterinfo(clustername, username) if not status: return [False, "cluster not found"] @@ -366,7 +369,7 @@ class VclusterMgr(object): # check gateway for user # after reboot, user gateway goes down and lose its configuration # so, check is necessary - self.networkmgr.check_usergw(username, self.nodemgr,self.distributedgw=='True') + self.networkmgr.check_usergw(username, uid, self.nodemgr,self.distributedgw=='True') # set proxy if not "proxy_server_ip" in info.keys(): info['proxy_server_ip'] = self.addr @@ -381,6 +384,7 @@ class VclusterMgr(object): except: return [False, "start cluster failed with setting proxy failed"] for container in info['containers']: + self.networkmgr.check_uservxlan(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT"))) if worker is None: return [False, "The worker can't be found or has been stopped."] @@ -402,14 +406,14 @@ class VclusterMgr(object): worker.mount_container(container['containername']) return [True, "mount cluster"] - def recover_cluster(self, clustername, username): + def recover_cluster(self, clustername, username, uid): [status, info] = self.get_clusterinfo(clustername, username) if not status: return [False, "cluster not found"] if info['status'] == 'stopped': return [True, "cluster no need to start"] # need to check and recover gateway of this user - self.networkmgr.check_usergw(username, self.nodemgr,self.distributedgw=='True') + self.networkmgr.check_usergw(username, uid, self.nodemgr,self.distributedgw=='True') # recover proxy of cluster if not "proxy_server_ip" in info.keys(): info['proxy_server_ip'] = self.addr @@ -425,10 +429,12 @@ class VclusterMgr(object): return [False, "start cluster failed with setting proxy failed"] # recover containers of this cluster for container in info['containers']: + self.networkmgr.check_uservxlan(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT"))) if worker is None: return [False, "The worker can't be found or has been stopped."] worker.recover_container(container['containername']) + #worker.check_usernet(uid, info['proxy_server_ip']) return [True, "start cluster"] # maybe here should use cluster id diff --git a/src/worker.py b/src/worker.py index c04463b..12358ed 100755 --- a/src/worker.py +++ b/src/worker.py @@ -127,6 +127,7 @@ class Worker(object): self.rpcserver.register_function(monitor.workerFetchInfo) self.rpcserver.register_function(netcontrol.setup_gw) self.rpcserver.register_function(netcontrol.del_gw) + self.rpcserver.register_function(ovscontrol.add_port_vxlan) self.rpcserver.register_function(netcontrol.check_gw) self.rpcserver.register_function(proxytool.set_route) self.rpcserver.register_function(proxytool.delete_route) diff --git a/user/user.py b/user/user.py index 54e20b5..c9a492f 100755 --- a/user/user.py +++ b/user/user.py @@ -320,6 +320,15 @@ def selfQuery_user(cur_user, user, form): result = G_usermgr.selfQuery(cur_user = cur_user) return json.dumps(result) +@app.route("/user/uid/", methods=['POST']) +@auth_key_required +def get_userid(): + username = request.form.get("username",None) + if username is None: + return json.dumps({'success':'false', 'message':'username field is required.'}) + else: + user = User.query.filter_by(username=username).first() + return json.dumps({'success':'true', 'uid':user.id}) @app.route("/user/selfModify/", methods=['POST']) @login_required From 9839097b5c1c813124e5e8b71fdf8705aac28dd1 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 9 May 2017 01:31:21 +0800 Subject: [PATCH 02/16] fix some bugs --- conf/lxc-script/lxc-ifup | 2 +- src/httprest.py | 6 +++--- src/nettools.py | 2 +- src/vclustermgr.py | 6 +++++- src/worker.py | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/conf/lxc-script/lxc-ifup b/conf/lxc-script/lxc-ifup index 5bcf1a8..e3df58a 100755 --- a/conf/lxc-script/lxc-ifup +++ b/conf/lxc-script/lxc-ifup @@ -8,7 +8,7 @@ # $5 : value of lxc.network.veth.pair ovs-vsctl --may-exist add-br $Bridge -if [ "$GatewayIP" != "NULL" ]; then +if [ "$GatewayHost" != "NULL" ]; then ovs-vsctl --may-exist add-port $Bridge vxlan-$UserID-$GatewayHost -- set interface vxlan-$UserID-$GatewayHost type=vxlan options:remote_ip=$GatewayHost options:key=$UserID fi ovs-vsctl --may-exist add-port $Bridge $5 diff --git a/src/httprest.py b/src/httprest.py index 5afad84..499107b 100755 --- a/src/httprest.py +++ b/src/httprest.py @@ -139,7 +139,7 @@ def create_cluster(user, beans, form): image['type'] = form.get("imagetype", None) image['owner'] = form.get("imageowner", None) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) - uid = user_info['id'] + uid = user_info['data']['id'] user_info = json.dumps(user_info) logger.info ("handle request : create cluster %s with image %s " % (clustername, image['name'])) setting = { @@ -174,7 +174,7 @@ def scaleout_cluster(user, beans, form): image['type'] = form.get("imagetype", None) image['owner'] = form.get("imageowner", None) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) - uid = user_info['id'] + uid = user_info['data']['id'] user_info = json.dumps(user_info) setting = { 'cpu': form.get('cpuSetting'), @@ -218,7 +218,7 @@ def start_cluster(user, beans, form): if (clustername == None): return json.dumps({'success':'false', 'message':'clustername is null'}) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) - uid = user_info['id'] + uid = user_info['data']['id'] logger.info ("handle request : start cluster %s" % clustername) [status, result] = G_vclustermgr.start_cluster(clustername, user, uid) if status: diff --git a/src/nettools.py b/src/nettools.py index 151fb55..11619a5 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import subprocess,network +import subprocess class ipcontrol(object): @staticmethod diff --git a/src/vclustermgr.py b/src/vclustermgr.py index 63f85f5..32a0882 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -4,10 +4,14 @@ import os, random, json, sys, imagemgr import datetime import xmlrpc.client -from httprest import post_to_user from log import logger import env import proxytool +import requests + +userpoint = "http://" + env.getenv('USER_IP') + ":" + str(env.getenv('USER_PORT')) +def post_to_user(url = '/', data={}): + return requests.post(userpoint+url,data=data).json() ################################################## # VclusterMgr diff --git a/src/worker.py b/src/worker.py index 12358ed..07fb8ef 100755 --- a/src/worker.py +++ b/src/worker.py @@ -14,7 +14,7 @@ import xmlrpc.server, sys, time from socketserver import ThreadingMixIn import threading import etcdlib, network, container -from nettools import netcontrol +from nettools import netcontrol,ovscontrol import monitor, proxytool from lvmtool import new_group, recover_group From e8269cc75668294c33508ebeb14965dd7be8d85f Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 9 May 2017 09:20:34 +0800 Subject: [PATCH 03/16] remove vlanid in usergw --- src/nettools.py | 8 ++++---- src/network.py | 10 +++++----- src/vclustermgr.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/nettools.py b/src/nettools.py index 11619a5..5165daa 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -250,8 +250,8 @@ class netcontrol(object): return ovscontrol.port_exists(gwport) @staticmethod - def setup_gw(bridge, gwport, addr, tag): - [status, result] = ovscontrol.add_port_internal_withtag(bridge, gwport, tag) + def setup_gw(bridge, gwport, addr): + [status, result] = ovscontrol.add_port_internal(bridge, gwport) if not status: return [status, result] [status, result] = ipcontrol.add_addr(gwport, addr) @@ -264,10 +264,10 @@ class netcontrol(object): return ovscontrol.del_port(bridge, gwport) @staticmethod - def check_gw(bridge, gwport, uid, addr, tag): + def check_gw(bridge, gwport, uid, addr): ovscontrol.add_bridge(bridge) if not netcontrol.gw_exists(bridge, gwport): - return netcontrol.setup_gw(bridge, gwport, addr, tag) + return netcontrol.setup_gw(bridge, gwport, addr) [status, info] = ipcontrol.link_info(gwport) if not status: return [False, "get gateway info failed"] diff --git a/src/network.py b/src/network.py index 350b4eb..93a9a1a 100755 --- a/src/network.py +++ b/src/network.py @@ -450,7 +450,7 @@ class NetworkMgr(object): self.load_usrgw(username) return username in self.usrgws.keys() - def setup_usrgw(self, username, nodemgr, workerip=None): + def setup_usrgw(self, username, uid, nodemgr, workerip=None): if not self.has_user(username): return [False,"user doesn't exist."] self.load_usrgw(username) @@ -464,12 +464,12 @@ class NetworkMgr(object): logger.info("setup gateway for %s with %s and vlan=%s on %s" % (username, usrpools.get_gateway_cidr(), str(usrpools.vlanid), ip)) self.usrgws[username] = ip self.dump_usrgw(username) - worker.setup_gw('docklet-br', username, usrpools.get_gateway_cidr(), str(usrpools.vlanid)) + worker.setup_gw('docklet-br-'+str(uid), username, usrpools.get_gateway_cidr()) else: logger.info("setup gateway for %s with %s and vlan=%s on master" % (username, usrpools.get_gateway_cidr(), str(usrpools.vlanid))) self.usrgws[username] = self.masterip self.dump_usrgw(username) - netcontrol.setup_gw('docklet-br', username, usrpools.get_gateway_cidr(), str(usrpools.vlanid)) + netcontrol.setup_gw('docklet-br-'+str(uid), username, usrpools.get_gateway_cidr()) self.dump_user(username) del self.users[username] return [True, "set up gateway success"] @@ -535,10 +535,10 @@ class NetworkMgr(object): self.del_usrgw(username,nodemgr) self.usrgws[username] = self.masterip self.dump_usrgw(username) - netcontrol.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr(), str(self.users[username].vlanid)) + netcontrol.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr()) else: worker = nodemgr.ip_to_rpc(ip) - worker.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr(), str(self.users[username].vlanid)) + worker.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr()) del self.users[username] return [True, 'check gw ok'] diff --git a/src/vclustermgr.py b/src/vclustermgr.py index 32a0882..88928da 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -103,7 +103,7 @@ class VclusterMgr(object): if not self.networkmgr.has_user(username): self.networkmgr.add_user(username, cidr=29, isshared = True if str(groupname) == "fundation" else False) if self.distributedgw == "False": - [success,message] = self.networkmgr.setup_usrgw(username, self.nodemgr) + [success,message] = self.networkmgr.setup_usrgw(username, uid, self.nodemgr) if not success: return [False, message] [status, result] = self.networkmgr.acquire_userips_cidr(username, clustersize) @@ -125,7 +125,7 @@ class VclusterMgr(object): workerip = workers[random.randint(0, len(workers)-1)] oneworker = xmlrpc.client.ServerProxy("http://%s:%s" % (workerip, env.getenv("WORKER_PORT"))) if self.distributedgw == "True" and i == 0 and not self.networkmgr.has_usrgw(username): - [success,message] = self.networkmgr.setup_usrgw(username, self.nodemgr, workerip) + [success,message] = self.networkmgr.setup_usrgw(username, uid, self.nodemgr, workerip) if not success: return [False, message] if i == 0: From 8400845dce17391b463776a2f53b305592b4c9d4 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 9 May 2017 09:24:46 +0800 Subject: [PATCH 04/16] fix a bug --- src/nettools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nettools.py b/src/nettools.py index 5165daa..a4dd5ff 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -277,7 +277,6 @@ class netcontrol(object): info['inet'].remove(addr) for otheraddr in info['inet']: ipcontrol.del_addr(gwport, otheraddr) - ovscontrol.set_port_tag(gwport, tag) if info['state'] == 'DOWN': ipcontrol.up_link(gwport) return [True, "check gateway port %s" % gwport] From c2d1d6e1245420989646c5703c5208c395b4096b Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 9 May 2017 10:03:24 +0800 Subject: [PATCH 05/16] start user before master in supermaster --- bin/docklet-supermaster | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/docklet-supermaster b/bin/docklet-supermaster index ed6369b..570713d 100755 --- a/bin/docklet-supermaster +++ b/bin/docklet-supermaster @@ -198,16 +198,16 @@ do_stop_user () { case "$1" in init) - do_start_master "new" + do_start_user do_start_proxy do_start_web - do_start_user + do_start_master "new" ;; start) - do_start_master "recovery" + do_start_user do_start_proxy do_start_web - do_start_user + do_start_master "recovery" ;; stop) @@ -218,14 +218,14 @@ case "$1" in ;; restart) + do_stop_user do_stop_web do_stop_proxy do_stop_master - do_stop_user - do_start_master "recovery" + do_start_user do_start_proxy do_start_web - do_start_user + do_start_master "recovery" ;; start_proxy) @@ -249,10 +249,10 @@ case "$1" in do_stop_proxy do_stop_master do_stop_user - do_start_master "new" + do_start_user do_start_proxy do_start_web - do_start_user + do_start_master "new" ;; status) From f9ac5106cc025256aa63b640406f00529e5db64e Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 9 May 2017 15:35:31 +0800 Subject: [PATCH 06/16] Remove GRE --- src/nodemgr.py | 4 ++-- src/worker.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nodemgr.py b/src/nodemgr.py index 264b442..e3578a8 100755 --- a/src/nodemgr.py +++ b/src/nodemgr.py @@ -109,14 +109,14 @@ class NodeMgr(object): logger.info ("new node %s joins" % nodeip) etcd_runip.append(nodeip) # setup GRE tunnels for new nodes - if self.addr == nodeip: + '''if self.addr == nodeip: logger.debug ("worker start on master node. not need to setup GRE") else: logger.debug ("setup GRE for %s" % nodeip) if netcontrol.gre_exists('docklet-br', nodeip): logger.debug("GRE for %s already exists, reuse it" % nodeip) else: - netcontrol.setup_gre('docklet-br', nodeip) + netcontrol.setup_gre('docklet-br', nodeip)''' self.etcd.setkey("machines/runnodes/"+nodeip, "ok") if nodeip not in self.runnodes: self.runnodes.append(nodeip) diff --git a/src/worker.py b/src/worker.py index 07fb8ef..b45f930 100755 --- a/src/worker.py +++ b/src/worker.py @@ -162,8 +162,8 @@ class Worker(object): sys.exit(1) logger.info ("setup GRE tunnel to master %s" % self.master) #network.netsetup("gre", self.master) - if not netcontrol.gre_exists('docklet-br', self.master): - netcontrol.setup_gre('docklet-br', self.master) + #if not netcontrol.gre_exists('docklet-br', self.master): + #netcontrol.setup_gre('docklet-br', self.master) # start service of worker def start(self): From a1295519f8a81eedba7db0db905344493b76b5c0 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Wed, 10 May 2017 17:38:31 +0800 Subject: [PATCH 07/16] Remove vlan and delete existing network when starting. --- src/httprest.py | 6 ++-- src/network.py | 73 +++++++++++++++++++++++++--------------------- src/nodemgr.py | 27 ++++++++--------- src/vclustermgr.py | 20 ++++++++----- src/worker.py | 16 ++++++---- 5 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/httprest.py b/src/httprest.py index 499107b..14d4d3b 100755 --- a/src/httprest.py +++ b/src/httprest.py @@ -139,7 +139,6 @@ def create_cluster(user, beans, form): image['type'] = form.get("imagetype", None) image['owner'] = form.get("imageowner", None) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) - uid = user_info['data']['id'] user_info = json.dumps(user_info) logger.info ("handle request : create cluster %s with image %s " % (clustername, image['name'])) setting = { @@ -152,7 +151,7 @@ def create_cluster(user, beans, form): result = res.get('result') if not status: return json.dumps({'success':'false', 'action':'create cluster', 'message':result}) - [status, result] = G_vclustermgr.create_cluster(clustername, user, uid, image, user_info, setting) + [status, result] = G_vclustermgr.create_cluster(clustername, user, image, user_info, setting) if status: return json.dumps({'success':'true', 'action':'create cluster', 'message':result}) else: @@ -174,7 +173,6 @@ def scaleout_cluster(user, beans, form): image['type'] = form.get("imagetype", None) image['owner'] = form.get("imageowner", None) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) - uid = user_info['data']['id'] user_info = json.dumps(user_info) setting = { 'cpu': form.get('cpuSetting'), @@ -186,7 +184,7 @@ def scaleout_cluster(user, beans, form): result = res.get('result') if not status: return json.dumps({'success':'false', 'action':'scale out', 'message': result}) - [status, result] = G_vclustermgr.scale_out_cluster(clustername, user, uid, image, user_info, setting) + [status, result] = G_vclustermgr.scale_out_cluster(clustername, user, image, user_info, setting) if status: return json.dumps({'success':'true', 'action':'scale out', 'message':result}) else: diff --git a/src/network.py b/src/network.py index 93a9a1a..8f0278a 100755 --- a/src/network.py +++ b/src/network.py @@ -239,16 +239,16 @@ class EnumPool(object): # wrap EnumPool with vlanid and gateway class UserPool(EnumPool): - def __init__(self, addr_cidr=None, vlanid=None, copy=None): - if addr_cidr and vlanid: + def __init__(self, addr_cidr=None, copy=None): + if addr_cidr: EnumPool.__init__(self, addr_cidr = addr_cidr) - self.vlanid=vlanid + #self.vlanid=vlanid self.pool.sort(key=ip_to_int) self.gateway = self.pool[0] self.pool.remove(self.gateway) elif copy: EnumPool.__init__(self, copy = copy) - self.vlanid = int(copy['vlanid']) + #self.vlanid = int(copy['vlanid']) self.gateway = copy['gateway'] else: logger.error("UserPool init failed with no addr_cidr or copy") @@ -268,7 +268,7 @@ class UserPool(EnumPool): return False def printpool(self): - print("users ID:"+str(self.vlanid)+", net info:"+self.info+", gateway:"+self.gateway) + print("net info:"+self.info+", gateway:"+self.gateway) print (str(self.pool)) # NetworkMgr : mange docklet network ip address @@ -293,9 +293,9 @@ class NetworkMgr(object): self.system = EnumPool(sysaddr+"/"+str(syscidr)) self.usrgws = {} self.users = {} - self.vlanids = {} - self.init_vlanids(4095, 60) - self.init_shared_vlanids() + #self.vlanids = {} + #self.init_vlanids(4095, 60) + #self.init_shared_vlanids() self.dump_center() self.dump_system() elif mode == 'recovery': @@ -304,15 +304,15 @@ class NetworkMgr(object): self.system = None self.usrgws = {} self.users = {} - self.vlanids = {} + #self.vlanids = {} self.load_center() self.load_system() - self.load_vlanids() - self.load_shared_vlanids() + #self.load_vlanids() + #self.load_shared_vlanids() else: logger.error("mode: %s not supported" % mode) - def init_vlanids(self, total, block): + '''def init_vlanids(self, total, block): self.vlanids['block'] = block self.etcd.setkey("network/vlanids/info", str(total)+"/"+str(block)) for i in range(1, int((total-1)/block)): @@ -320,11 +320,11 @@ class NetworkMgr(object): self.vlanids['currentpool'] = list(range(1+block*i, total+1)) self.vlanids['currentindex'] = i+1 self.etcd.setkey("network/vlanids/"+str(i+1), json.dumps(self.vlanids['currentpool'])) - self.etcd.setkey("network/vlanids/current", str(i+1)) + self.etcd.setkey("network/vlanids/current", str(i+1))''' # Data Structure: # shared_vlanids = [{vlanid = ..., sharenum = ...}, {vlanid = ..., sharenum = ...}, ...] - def init_shared_vlanids(self, vlannum = 128, sharenum = 128): + '''def init_shared_vlanids(self, vlannum = 128, sharenum = 128): self.shared_vlanids = [] for i in range(vlannum): shared_vlanid = {} @@ -364,7 +364,7 @@ class NetworkMgr(object): self.shared_vlanids = json.loads(shared_vlanids) def dump_shared_vlanids(self): - self.etcd.setkey("network/shared_vlanids", json.dumps(self.shared_vlanids)) + self.etcd.setkey("network/shared_vlanids", json.dumps(self.shared_vlanids))''' def load_center(self): [status, centerdata] = self.etcd.getkey("network/center") @@ -389,7 +389,7 @@ class NetworkMgr(object): self.users[username] = user def dump_user(self, username): - self.etcd.setkey("network/users/"+username, json.dumps({'info':self.users[username].info, 'vlanid':self.users[username].vlanid, 'gateway':self.users[username].gateway, 'pool':self.users[username].pool})) + self.etcd.setkey("network/users/"+username, json.dumps({'info':self.users[username].info, 'gateway':self.users[username].gateway, 'pool':self.users[username].pool})) def load_usrgw(self,username): [status, data] = self.etcd.getkey("network/usrgws/"+username) @@ -406,10 +406,10 @@ class NetworkMgr(object): self.system.printpool() print ("") print (" users in users is in etcd, not in memory") - print ("") - print (str(self.vlanids['currentindex'])+":"+str(self.vlanids['currentpool'])) + #print ("") + #print (str(self.vlanids['currentindex'])+":"+str(self.vlanids['currentpool'])) - def acquire_vlanid(self, isshared = False): + '''def acquire_vlanid(self, isshared = False): if isshared: # only share vlanid of the front entry # if sharenum is reduced to 0, move the front entry to the back @@ -444,7 +444,7 @@ class NetworkMgr(object): else: self.vlanids['currentpool'].append(vlanid) self.dump_vlanids() - return [True, "Release VLAN ID success"] + return [True, "Release VLAN ID success"]''' def has_usrgw(self, username): self.load_usrgw(username) @@ -461,12 +461,12 @@ class NetworkMgr(object): if(workerip is not None): ip = workerip worker = nodemgr.ip_to_rpc(workerip) - logger.info("setup gateway for %s with %s and vlan=%s on %s" % (username, usrpools.get_gateway_cidr(), str(usrpools.vlanid), ip)) + logger.info("setup gateway for %s with %s on %s" % (username, usrpools.get_gateway_cidr(), ip)) self.usrgws[username] = ip self.dump_usrgw(username) worker.setup_gw('docklet-br-'+str(uid), username, usrpools.get_gateway_cidr()) else: - logger.info("setup gateway for %s with %s and vlan=%s on master" % (username, usrpools.get_gateway_cidr(), str(usrpools.vlanid))) + logger.info("setup gateway for %s with %s on master" % (username, usrpools.get_gateway_cidr() )) self.usrgws[username] = self.masterip self.dump_usrgw(username) netcontrol.setup_gw('docklet-br-'+str(uid), username, usrpools.get_gateway_cidr()) @@ -482,34 +482,37 @@ class NetworkMgr(object): self.dump_center() if status == False: return [False, result] - [status, vlanid] = self.acquire_vlanid(isshared) + '''[status, vlanid] = self.acquire_vlanid(isshared) if status: vlanid = int(vlanid) else: self.center.free(result, cidr) self.dump_center() - return [False, vlanid] - self.users[username] = UserPool(addr_cidr = result+"/"+str(cidr), vlanid=vlanid) + return [False, vlanid]''' + self.users[username] = UserPool(addr_cidr = result+"/"+str(cidr)) #logger.info("setup gateway for %s with %s and vlan=%s" % (username, self.users[username].get_gateway_cidr(), str(vlanid))) #netcontrol.setup_gw('docklet-br', username, self.users[username].get_gateway_cidr(), str(vlanid)) self.dump_user(username) del self.users[username] return [True, 'add user success'] - def del_usrgw(self, username, nodemgr): + def del_usrgwbr(self, username, uid, nodemgr): if username not in self.usrgws.keys(): return [False, "user does't have gateway or user doesn't exist."] ip = self.usrgws[username] + logger.info("Delete user %s(%s) gateway on %s" %(username, str(uid), ip)) if ip == self.masterip: - netcontrol.del_gw('docklet-br', username) + netcontrol.del_gw('docklet-br-'+str(uid), username) + netcontrol.del_bridge('docklet-br-'+str(uid)) else: worker = nodemgr.ip_to_rpc(ip) - worker.del_gw('docklet-br', username) + worker.del_gw('docklet-br-'+str(uid), username) + worker.del_bridge('docklet-br-'+str(uid)) del self.usrgws[username] self.etcd.delkey("network/usrgws/"+username) return [True, 'delete user\' gateway success'] - def del_user(self, username, isshared = False): + def del_user(self, username): if not self.has_user(username): return [False, username+" not in users set"] self.load_user(username) @@ -517,14 +520,15 @@ class NetworkMgr(object): logger.info ("delete user %s with cidr=%s" % (username, int(cidr))) self.center.free(addr, int(cidr)) self.dump_center() - if not isshared: - self.release_vlanid(self.users[username].vlanid) + #if not isshared: + #self.release_vlanid(self.users[username].vlanid) #netcontrol.del_gw('docklet-br', username) self.etcd.deldir("network/users/"+username) del self.users[username] return [True, 'delete user success'] def check_usergw(self, username, uid, nodemgr, distributedgw=False): + logger.info("Check %s(%s) user gateway."%(username, str(uid))) self.load_usrgw(username) if username not in self.usrgws.keys(): return [False, 'user does not exist.'] @@ -532,7 +536,7 @@ class NetworkMgr(object): self.load_user(username) if not distributedgw: if not ip == self.masterip: - self.del_usrgw(username,nodemgr) + self.del_usrgwbr(username,uid,nodemgr) self.usrgws[username] = self.masterip self.dump_usrgw(username) netcontrol.check_gw('docklet-br-'+str(uid), username, uid, self.users[username].get_gateway_cidr()) @@ -543,6 +547,7 @@ class NetworkMgr(object): return [True, 'check gw ok'] def check_uservxlan(self, username, uid, remote, nodemgr, distributedgw=False): + logger.info("Check %s(%s) vxlan from gateway host to %s." % (username, str(uid), remote)) self.load_usrgw(username) if username not in self.usrgws.keys(): return [False, 'user does not exist.'] @@ -609,14 +614,14 @@ class NetworkMgr(object): del self.users[username] return result - def get_uservlanid(self, username): + '''def get_uservlanid(self, username): if not self.has_user(username): return [False, 'username not exists in users set'] self.load_user(username) result = self.users[username].vlanid self.dump_user(username) del self.users[username] - return result + return result''' def acquire_sysips(self, num=1): logger.info ("acquire system ips") diff --git a/src/nodemgr.py b/src/nodemgr.py index e3578a8..84655cf 100755 --- a/src/nodemgr.py +++ b/src/nodemgr.py @@ -2,7 +2,7 @@ import threading, random, time, xmlrpc.client, sys #import network -from nettools import netcontrol +from nettools import netcontrol,ovscontrol from log import logger import env @@ -24,27 +24,24 @@ class NodeMgr(object): self.mode = mode self.workerport = env.getenv('WORKER_PORT') - # initialize the network - logger.info ("initialize network") + # delete the existing network + logger.info ("delete the existing network") + [success, bridges] = ovscontrol.list_bridges() + if success: + for bridge in bridges: + if bridge.startswith("docklet-br"): + ovscontrol.del_bridge(bridge) + else: + logger.error(bridges) - # 'docklet-br' not need ip address. Because every user has gateway - #[status, result] = self.networkmgr.acquire_sysips_cidr() - #self.networkmgr.printpools() - #if not status: - # logger.info ("initialize network failed, no IP for system bridge") - # sys.exit(1) - #self.bridgeip = result[0] - #logger.info ("initialize bridge wih ip %s" % self.bridgeip) - #network.netsetup("init", self.bridgeip) - - if self.mode == 'new': + '''if self.mode == 'new': if netcontrol.bridge_exists('docklet-br'): netcontrol.del_bridge('docklet-br') netcontrol.new_bridge('docklet-br') else: if not netcontrol.bridge_exists('docklet-br'): logger.error("docklet-br not found") - sys.exit(1) + sys.exit(1)''' # get allnodes self.allnodes = self._nodelist_etcd("allnodes") diff --git a/src/vclustermgr.py b/src/vclustermgr.py index 88928da..51c467b 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -88,7 +88,7 @@ class VclusterMgr(object): self.detach_cluster(cluster, user) logger.info("detached all vclusters for all users") - def create_cluster(self, clustername, username, uid, image, user_info, setting): + def create_cluster(self, clustername, username, image, user_info, setting): if self.is_cluster(clustername, username): return [False, "cluster:%s already exists" % clustername] clustersize = int(self.defaultsize) @@ -96,6 +96,7 @@ class VclusterMgr(object): workers = self.nodemgr.get_nodeips() image_json = json.dumps(image) groupname = json.loads(user_info)["data"]["group"] + uid = json.loads(user_info)["data"]["id"] if (len(workers) == 0): logger.warning ("no workers to start containers, start cluster failed") return [False, "no workers are running"] @@ -108,7 +109,7 @@ class VclusterMgr(object): return [False, message] [status, result] = self.networkmgr.acquire_userips_cidr(username, clustersize) gateway = self.networkmgr.get_usergw(username) - vlanid = self.networkmgr.get_uservlanid(username) + #vlanid = self.networkmgr.get_uservlanid(username) logger.info ("create cluster with gateway : %s" % gateway) self.networkmgr.printpools() if not status: @@ -151,7 +152,7 @@ class VclusterMgr(object): clusterfile.close() return [True, info] - def scale_out_cluster(self,clustername,username,uid, image,user_info, setting): + def scale_out_cluster(self,clustername,username, image,user_info, setting): if not self.is_cluster(clustername,username): return [False, "cluster:%s not found" % clustername] workers = self.nodemgr.get_nodeips() @@ -161,7 +162,7 @@ class VclusterMgr(object): image_json = json.dumps(image) [status, result] = self.networkmgr.acquire_userips_cidr(username) gateway = self.networkmgr.get_usergw(username) - vlanid = self.networkmgr.get_uservlanid(username) + #vlanid = self.networkmgr.get_uservlanid(username) self.networkmgr.printpools() if not status: return [False, result] @@ -176,7 +177,8 @@ class VclusterMgr(object): lxc_name = username + "-" + str(clusterid) + "-" + str(cid) hostname = "host-" + str(cid) proxy_server_ip = clusterinfo['proxy_server_ip'] - [success, message] = oneworker.create_container(lxc_name, username, uid, json.dumps(setting), clustername, clusterid, str(cid), hostname, ip, gateway, image_json) + uid = json.loads(user_info)["data"]["id"] + [success, message] = oneworker.create_container(lxc_name, proxy_server_ip, username, uid, json.dumps(setting), clustername, clusterid, str(cid), hostname, ip, gateway, image_json) if success is False: logger.info("create container failed, so scale out failed") return [False, message] @@ -315,11 +317,12 @@ class VclusterMgr(object): os.remove(self.fspath+"/global/users/"+username+"/hosts/"+str(info['clusterid'])+".hosts") groupname = json.loads(user_info)["data"]["group"] + uid = json.loads(user_info)["data"]["id"] [status, clusters] = self.list_clusters(username) if len(clusters) == 0: - self.networkmgr.del_user(username, isshared = True if str(groupname) == "fundation" else False) - self.networkmgr.del_usrgw(username, self.nodemgr) - logger.info("vlanid release triggered") + self.networkmgr.del_user(username) + self.networkmgr.del_usrgwbr(username, uid, self.nodemgr) + #logger.info("vlanid release triggered") return [True, "cluster delete"] @@ -388,6 +391,7 @@ class VclusterMgr(object): except: return [False, "start cluster failed with setting proxy failed"] for container in info['containers']: + # set up vxlan from user's gateway host to container's host. self.networkmgr.check_uservxlan(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT"))) if worker is None: diff --git a/src/worker.py b/src/worker.py index b45f930..616302f 100755 --- a/src/worker.py +++ b/src/worker.py @@ -127,6 +127,7 @@ class Worker(object): self.rpcserver.register_function(monitor.workerFetchInfo) self.rpcserver.register_function(netcontrol.setup_gw) self.rpcserver.register_function(netcontrol.del_gw) + self.rpcserver.register_function(netcontrol.del_bridge) self.rpcserver.register_function(ovscontrol.add_port_vxlan) self.rpcserver.register_function(netcontrol.check_gw) self.rpcserver.register_function(proxytool.set_route) @@ -138,10 +139,15 @@ class Worker(object): self.con_collector = monitor.Container_Collector() self.hosts_collector = monitor.Collector() - # initialize the network - # if worker and master run on the same node, reuse bridges - # don't need to create new bridges - if (self.addr == self.master): + # delete the existing network + [success, bridges] = ovscontrol.list_bridges() + if success: + for bridge in bridges: + if bridge.startswith("docklet-br"): + ovscontrol.del_bridge(bridge) + else: + logger.error(bridges) + '''if (self.addr == self.master): logger.info ("master also on this node. reuse master's network") else: logger.info ("initialize network") @@ -163,7 +169,7 @@ class Worker(object): logger.info ("setup GRE tunnel to master %s" % self.master) #network.netsetup("gre", self.master) #if not netcontrol.gre_exists('docklet-br', self.master): - #netcontrol.setup_gre('docklet-br', self.master) + #netcontrol.setup_gre('docklet-br', self.master)''' # start service of worker def start(self): From 0b733b79f526491fc75e88a4a7e2f3c598b7fdab Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Wed, 10 May 2017 18:40:59 +0800 Subject: [PATCH 08/16] add some comment --- src/vclustermgr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vclustermgr.py b/src/vclustermgr.py index 51c467b..597e77b 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -437,6 +437,7 @@ class VclusterMgr(object): return [False, "start cluster failed with setting proxy failed"] # recover containers of this cluster for container in info['containers']: + # set up vxlan from user's gateway host to container's host. self.networkmgr.check_uservxlan(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT"))) if worker is None: From eb7e5f826fc5c5ad8ebf6c7e66d808f4c4355bfd Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Thu, 11 May 2017 17:24:44 +0800 Subject: [PATCH 09/16] recover usernet when recovering cluster --- src/nettools.py | 16 ++++++++++++++-- src/vclustermgr.py | 4 +++- src/worker.py | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/nettools.py b/src/nettools.py index a4dd5ff..86ef539 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -182,6 +182,14 @@ class ovscontrol(object): except subprocess.CalledProcessError as suberror: return [False, "delete port failed : %s" % suberror.stdout.decode('utf-8')] + @staticmethod + def add_port(bridge, port): + try: + subprocess.run(['ovs-vsctl', '--may-exist', 'add-port', str(bridge), str(port)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True) + return [True, str(port)] + except subprocess.CalledProcessError as suberror: + return [False, "add port failed : %s" % suberror.stdout.decode('utf-8')] + @staticmethod def add_port_internal(bridge, port): try: @@ -281,5 +289,9 @@ class netcontrol(object): ipcontrol.up_link(gwport) return [True, "check gateway port %s" % gwport] - #@staticmethod - #def check_usernet(uid, GatewayHost): + @staticmethod + def recover_usernet(portname, uid, GatewayHost, isGatewayHost): + ovscontrol.add_bridge("docklet-br-"+str(uid)) + if not isGatewayHost: + ovscontrol.add_port_vxlan("docklet-br-"+str(uid), "vxlan-"+str(uid)+"-"+GatewayHost, GatewayHost, str(uid)) + ovscontrol.add_port("docklet-br-"+str(uid), portname) diff --git a/src/vclustermgr.py b/src/vclustermgr.py index 597e77b..aaafcd3 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -443,7 +443,9 @@ class VclusterMgr(object): if worker is None: return [False, "The worker can't be found or has been stopped."] worker.recover_container(container['containername']) - #worker.check_usernet(uid, info['proxy_server_ip']) + namesplit = container['containername'].split('-') + portname = namesplit[1] + '-' + namesplit[2] + worker.recover_usernet(portname, uid, info['proxy_server_ip'], container['host']==info['proxy_server_ip']) return [True, "start cluster"] # maybe here should use cluster id diff --git a/src/worker.py b/src/worker.py index 616302f..1ea46f4 100755 --- a/src/worker.py +++ b/src/worker.py @@ -130,6 +130,7 @@ class Worker(object): self.rpcserver.register_function(netcontrol.del_bridge) self.rpcserver.register_function(ovscontrol.add_port_vxlan) self.rpcserver.register_function(netcontrol.check_gw) + self.rpcserver.register_function(netcontrol.recover_usernet) self.rpcserver.register_function(proxytool.set_route) self.rpcserver.register_function(proxytool.delete_route) # register functions or instances to server for rpc From 08c5c780c0875012c6046c57a4f5d5066398cdf9 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 16 May 2017 10:10:09 +0800 Subject: [PATCH 10/16] change network config in container.conf --- conf/container.conf | 2 +- conf/lxc-script/lxc-ifup | 3 --- src/container.py | 6 ------ src/vclustermgr.py | 3 +++ tools/update_con_network.py | 8 ++++++++ 5 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 tools/update_con_network.py diff --git a/conf/container.conf b/conf/container.conf index 1f3db65..ab1a7a8 100644 --- a/conf/container.conf +++ b/conf/container.conf @@ -25,7 +25,7 @@ lxc.network.type = veth lxc.network.name = eth0 # veth.pair is limited in 16 bytes lxc.network.veth.pair = %VETHPAIR% -lxc.network.script.up = Bridge=docklet-br-%UserID% GatewayHost=%GatewayHost% UserID=%UserID% %LXCSCRIPT%/lxc-ifup +lxc.network.script.up = Bridge=docklet-br-%UserID% %LXCSCRIPT%/lxc-ifup lxc.network.script.down = Bridge=docklet-br-%UserID% %LXCSCRIPT%/lxc-ifdown lxc.network.ipv4 = %IP% lxc.network.ipv4.gateway = %GATEWAY% diff --git a/conf/lxc-script/lxc-ifup b/conf/lxc-script/lxc-ifup index e3df58a..17dacf7 100755 --- a/conf/lxc-script/lxc-ifup +++ b/conf/lxc-script/lxc-ifup @@ -8,7 +8,4 @@ # $5 : value of lxc.network.veth.pair ovs-vsctl --may-exist add-br $Bridge -if [ "$GatewayHost" != "NULL" ]; then - ovs-vsctl --may-exist add-port $Bridge vxlan-$UserID-$GatewayHost -- set interface vxlan-$UserID-$GatewayHost type=vxlan options:remote_ip=$GatewayHost options:key=$UserID -fi ovs-vsctl --may-exist add-port $Bridge $5 diff --git a/src/container.py b/src/container.py index 20edbbd..9512f63 100755 --- a/src/container.py +++ b/src/container.py @@ -63,12 +63,6 @@ class Container(object): content = content.replace("%LXCSCRIPT%",env.getenv("LXC_SCRIPT")) content = content.replace("%LXCNAME%",lxc_name) content = content.replace("%UserID%",str(uid)) - net_dev = env.getenv("NETWORK_DEVICE") - ipaddr = network.getip(net_dev) - if ipaddr == proxy_server_ip: - content = content.replace("%GatewayHost%","NULL") - else: - content = content.replace("%GatewayHost%",proxy_server_ip) content = content.replace("%CLUSTERNAME%", clustername) content = content.replace("%VETHPAIR%", str(clusterid)+'-'+str(containerid)) return content diff --git a/src/vclustermgr.py b/src/vclustermgr.py index a19f5fb..6bc8b7a 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -398,6 +398,9 @@ class VclusterMgr(object): return [False, "The worker can't be found or has been stopped."] worker.start_container(container['containername']) worker.start_services(container['containername']) + namesplit = container['containername'].split('-') + portname = namesplit[1] + '-' + namesplit[2] + worker.recover_usernet(portname, uid, info['proxy_server_ip'], container['host']==info['proxy_server_ip']) info['status']='running' info['start_time']=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.write_clusterinfo(info,clustername,username) diff --git a/tools/update_con_network.py b/tools/update_con_network.py new file mode 100644 index 0000000..6a522b1 --- /dev/null +++ b/tools/update_con_network.py @@ -0,0 +1,8 @@ + + + +userpoint = "http://" + env.getenv('USER_IP') + ":" + str(env.getenv('USER_PORT')) +G_userip = env.getenv("USER_IP") + +def post_to_user(url = '/', data={}): + return requests.post(userpoint+url,data=data).json() From a18398dfb6a5ffb021d78789eb2253259c71bfe5 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Tue, 16 May 2017 11:15:23 +0800 Subject: [PATCH 11/16] add a script to update network in container config --- tools/update_con_network.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tools/update_con_network.py b/tools/update_con_network.py index 6a522b1..0400893 100644 --- a/tools/update_con_network.py +++ b/tools/update_con_network.py @@ -1,8 +1,26 @@ - - +import sys,os +sys.path.append("../src/") +import env,requests userpoint = "http://" + env.getenv('USER_IP') + ":" + str(env.getenv('USER_PORT')) G_userip = env.getenv("USER_IP") +auth_key = env.getenv('AUTH_KEY') def post_to_user(url = '/', data={}): return requests.post(userpoint+url,data=data).json() + +cons = os.listdir('/var/lib/lxc') +for con in cons: + print("Update %s..."%(con)) + namesplit = con.split('-') + user = namesplit[0] + res = post_to_user('/user/uid/',{'username':user,'auth_key':auth_key}) + configfile = open('/var/lib/lxc/'+con+'/config','r') + context = configfile.read() + configfile.close() + #print(context) + #print(res['uid']) + context = context.replace("docklet-br","docklet-br-"+str(res['uid'])) + newfile = open('/var/lib/lxc/'+con+'/config','w') + newfile.write(context) + newfile.close() From f97e42426822b7bc70178189094b013cb5ecdf18 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Wed, 17 May 2017 19:47:29 +0800 Subject: [PATCH 12/16] add ovscontrol to network --- src/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network.py b/src/network.py index 8f0278a..b73295a 100755 --- a/src/network.py +++ b/src/network.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 import json, sys, netifaces -from nettools import netcontrol +from nettools import netcontrol,ovscontrol from log import logger From 9651d05d4747a8165c3f3c640b704a0b286dd1b3 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Fri, 19 May 2017 21:52:43 +0800 Subject: [PATCH 13/16] replace vxlan with gre --- src/nettools.py | 6 +++--- src/network.py | 10 +++++----- src/vclustermgr.py | 8 ++++---- src/worker.py | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/nettools.py b/src/nettools.py index 86ef539..6351fb8 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -215,9 +215,9 @@ class ovscontrol(object): return [False, "add port failed : %s" % suberror.stdout.decode('utf-8')] @staticmethod - def add_port_vxlan(bridge, port, remote, key): + def add_port_gre_withkey(bridge, port, remote, key): try: - subprocess.run(['ovs-vsctl', '--may-exist', 'add-port', str(bridge), str(port), '--', 'set', 'interface', str(port), 'type=vxlan', 'options:remote_ip='+str(remote), 'options:key='+str(key)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True) + subprocess.run(['ovs-vsctl', '--may-exist', 'add-port', str(bridge), str(port), '--', 'set', 'interface', str(port), 'type=gre', 'options:remote_ip='+str(remote), 'options:key='+str(key)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True) return [True, str(port)] except subprocess.CalledProcessError as suberror: return [False, "add port failed : %s" % suberror.stdout.decode('utf-8')] @@ -293,5 +293,5 @@ class netcontrol(object): def recover_usernet(portname, uid, GatewayHost, isGatewayHost): ovscontrol.add_bridge("docklet-br-"+str(uid)) if not isGatewayHost: - ovscontrol.add_port_vxlan("docklet-br-"+str(uid), "vxlan-"+str(uid)+"-"+GatewayHost, GatewayHost, str(uid)) + ovscontrol.add_port_gre_withkey("docklet-br-"+str(uid), "gre-"+str(uid)+"-"+GatewayHost, GatewayHost, str(uid)) ovscontrol.add_port("docklet-br-"+str(uid), portname) diff --git a/src/network.py b/src/network.py index b73295a..c4c3086 100755 --- a/src/network.py +++ b/src/network.py @@ -546,20 +546,20 @@ class NetworkMgr(object): del self.users[username] return [True, 'check gw ok'] - def check_uservxlan(self, username, uid, remote, nodemgr, distributedgw=False): - logger.info("Check %s(%s) vxlan from gateway host to %s." % (username, str(uid), remote)) + def check_usergre(self, username, uid, remote, nodemgr, distributedgw=False): + logger.info("Check %s(%s) gre from gateway host to %s." % (username, str(uid), remote)) self.load_usrgw(username) if username not in self.usrgws.keys(): return [False, 'user does not exist.'] ip = self.usrgws[username] if not distributedgw: if not remote == self.masterip: - ovscontrol.add_port_vxlan('docklet-br-'+str(uid), 'vxlan-'+str(uid)+'-'+remote, remote, uid) + ovscontrol.add_port_gre_withkey('docklet-br-'+str(uid), 'gre-'+str(uid)+'-'+remote, remote, uid) else: if not remote == ip: worker = nodemgr.ip_to_rpc(ip) - worker.add_port_vxlan('docklet-br-'+str(uid), 'vxlan-'+str(uid)+'-'+remote, remote, uid) - return [True, 'check vxlan ok'] + worker.add_port_gre_withkey('docklet-br-'+str(uid), 'gre-'+str(uid)+'-'+remote, remote, uid) + return [True, 'check gre ok'] def has_user(self, username): [status, _value] = self.etcd.getkey("network/users/"+username) diff --git a/src/vclustermgr.py b/src/vclustermgr.py index bd4dc92..f6378c7 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -431,8 +431,8 @@ class VclusterMgr(object): except: return [False, "start cluster failed with setting proxy failed"] for container in info['containers']: - # set up vxlan from user's gateway host to container's host. - self.networkmgr.check_uservxlan(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') + # set up gre from user's gateway host to container's host. + self.networkmgr.check_usergre(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT"))) if worker is None: return [False, "The worker can't be found or has been stopped."] @@ -492,8 +492,8 @@ class VclusterMgr(object): return [False, "start cluster failed with setting proxy failed"] # recover containers of this cluster for container in info['containers']: - # set up vxlan from user's gateway host to container's host. - self.networkmgr.check_uservxlan(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') + # set up gre from user's gateway host to container's host. + self.networkmgr.check_usergre(username, uid, container['host'], self.nodemgr, self.distributedgw=='True') worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT"))) if worker is None: return [False, "The worker can't be found or has been stopped."] diff --git a/src/worker.py b/src/worker.py index 600b960..29cef83 100755 --- a/src/worker.py +++ b/src/worker.py @@ -128,7 +128,7 @@ class Worker(object): self.rpcserver.register_function(netcontrol.setup_gw) self.rpcserver.register_function(netcontrol.del_gw) self.rpcserver.register_function(netcontrol.del_bridge) - self.rpcserver.register_function(ovscontrol.add_port_vxlan) + self.rpcserver.register_function(ovscontrol.add_port_gre_withkey) self.rpcserver.register_function(netcontrol.check_gw) self.rpcserver.register_function(netcontrol.recover_usernet) self.rpcserver.register_function(proxytool.set_route) @@ -199,7 +199,7 @@ class Worker(object): else: logger.error("get key %s failed, master may be crashed" % self.addr) self.etcd.setkey("machines/runnodes/"+self.addr, "ok", ttl = 60) - + if __name__ == '__main__': From 5b6586a43ecd1d8fecc9964fbf0fb94d22784646 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Fri, 19 May 2017 22:11:00 +0800 Subject: [PATCH 14/16] update update_con_network.py --- tools/update_con_network.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/update_con_network.py b/tools/update_con_network.py index 0400893..1677b1c 100644 --- a/tools/update_con_network.py +++ b/tools/update_con_network.py @@ -2,8 +2,11 @@ import sys,os sys.path.append("../src/") import env,requests -userpoint = "http://" + env.getenv('USER_IP') + ":" + str(env.getenv('USER_PORT')) -G_userip = env.getenv("USER_IP") +if len(sys.argv) < 2: + print("Please enter USER_IP") + exit() + +userpoint = "http://" + sys.argv[1] + ":" + str(env.getenv('USER_PORT')) auth_key = env.getenv('AUTH_KEY') def post_to_user(url = '/', data={}): @@ -15,7 +18,10 @@ for con in cons: namesplit = con.split('-') user = namesplit[0] res = post_to_user('/user/uid/',{'username':user,'auth_key':auth_key}) - configfile = open('/var/lib/lxc/'+con+'/config','r') + try: + configfile = open('/var/lib/lxc/'+con+'/config','r') + except: + continue context = configfile.read() configfile.close() #print(context) From 7692fbb3ea346bc804c9b4343f78d75364ef212f Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Sun, 21 May 2017 13:40:59 +0800 Subject: [PATCH 15/16] build usernet for new node when adding nodes to running clusters --- src/vclustermgr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vclustermgr.py b/src/vclustermgr.py index f6378c7..8da6a4f 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -186,8 +186,12 @@ class VclusterMgr(object): logger.info("create container failed, so scale out failed") return [False, message] if clusterinfo['status'] == "running": + self.networkmgr.check_usergre(username, uid, workerip, self.nodemgr, self.distributedgw=='True') oneworker.start_container(lxc_name) - oneworker.start_services(lxc_name, ["ssh"]) # TODO: need fix + oneworker.start_services(lxc_name, ["ssh"]) # TODO: need fix + namesplit = lxc_name.split('-') + portname = namesplit[1] + '-' + namesplit[2] + oneworker.recover_usernet(portname, uid, proxy_server_ip, workerip==proxy_server_ip) logger.info("scale out success") hostfile = open(hostpath, 'a') hostfile.write(ip.split("/")[0] + "\t" + hostname + "\t" + hostname + "." + clustername + "\n") From 589ebefc3c65f7fead8ed95d1a42cd93d75ed262 Mon Sep 17 00:00:00 2001 From: zhuyj17 Date: Sun, 21 May 2017 13:47:58 +0800 Subject: [PATCH 16/16] Delete old gre when moving gateway to master. --- src/nettools.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nettools.py b/src/nettools.py index 6351fb8..c43ba21 100755 --- a/src/nettools.py +++ b/src/nettools.py @@ -292,6 +292,11 @@ class netcontrol(object): @staticmethod def recover_usernet(portname, uid, GatewayHost, isGatewayHost): ovscontrol.add_bridge("docklet-br-"+str(uid)) + [success, ports] = ovscontrol.list_ports("docklet-br-"+str(uid)) + if success: + for port in ports: + if port.startswith("gre"): + ovscontrol.del_port("docklet-br-"+str(uid),port) if not isGatewayHost: ovscontrol.add_port_gre_withkey("docklet-br-"+str(uid), "gre-"+str(uid)+"-"+GatewayHost, GatewayHost, str(uid)) ovscontrol.add_port("docklet-br-"+str(uid), portname)