Merge pull request #267 from FirmlyReality/portmapping

Portmapping
This commit is contained in:
Yujian Zhu 2017-07-17 11:32:23 +08:00 committed by GitHub
commit f987ad2d5b
10 changed files with 347 additions and 35 deletions

View File

@ -172,3 +172,9 @@
# or to request master from users server. Please set the
# same value on each machine. Please don't use the default value.
# AUTH_KEY=docklet
# ALLOCATED_PORTS: the ports on this host that will be allocated to users.
# The allocated ports are for ports mapping. Default: 10000-65535
# The two ports next to '-' are inclueded. If there are several ranges,
# Please seperate them by ',' , for example: 10000-20000,30000-40000
# ALLOCATED_PORTS=10000-65535

View File

@ -14,7 +14,7 @@ fi
# install packages that docklet needs (in ubuntu)
# some packages' name maybe different in debian
apt-get install -y cgmanager lxc lxcfs lxc-templates lvm2 bridge-utils curl exim4 openssh-server openvswitch-switch
apt-get install -y cgmanager lxc lxcfs lxc-templates lvm2 bridge-utils curl exim4 openssh-server openvswitch-switch
apt-get install -y python3 python3-netifaces python3-flask python3-flask-sqlalchemy python3-pampy python3-httplib2
apt-get install -y python3-psutil
apt-get install -y python3-lxc
@ -24,6 +24,10 @@ apt-get install -y etcd
apt-get install -y glusterfs-client
apt-get install -y nginx
#add ip forward
echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf
sysctl -p
# check cgroup control
which cgm &> /dev/null || { echo "FAILED : cgmanager is required, please install cgmanager" && exit 1; }
cpucontrol=$(cgm listkeys cpu)
@ -75,4 +79,3 @@ echo ""
echo "Then start docklet as described in README.md"

View File

@ -75,5 +75,7 @@ def getenv(key):
return os.environ.get("OPEN_REGISTRY","False")
elif key =="APPROVAL_RBT":
return os.environ.get("APPROVAL_RBT","ON")
elif key =="ALLOCATED_PORTS":
return os.environ.get("ALLOCATED_PORTS","10000-65535")
else:
return os.environ.get(key,"")

View File

@ -30,6 +30,7 @@ import monitor,traceback
import threading
import sysmgr
import requests
from nettools import portcontrol
#default EXTERNAL_LOGIN=False
external_login = env.getenv('EXTERNAL_LOGIN')
@ -397,6 +398,38 @@ def deleteproxy(user, beans, form):
G_vclustermgr.deleteproxy(user,clustername)
return json.dumps({'success':'true', 'action':'deleteproxy'})
@app.route("/port_mapping/add/", methods=['POST'])
@login_required
def add_port_mapping(user, beans, form):
global G_vclustermgr
logger.info ("handle request : add port mapping")
node_name = form.get("node_name",None)
node_ip = form.get("node_ip", None)
node_port = form.get("node_port", None)
clustername = form.get("clustername", None)
if node_name is None or node_ip is None or node_port is None or clustername is None:
return json.dumps({'success':'false', 'message': 'Illegal form.'})
[status, message] = G_vclustermgr.add_port_mapping(user,clustername,node_name,node_ip,node_port)
if status is True:
return json.dumps({'success':'true', 'action':'addproxy'})
else:
return json.dumps({'success':'false', 'message': message})
@app.route("/port_mapping/delete/", methods=['POST'])
@login_required
def delete_port_mapping(user, beans, form):
global G_vclustermgr
logger.info ("handle request : delete port mapping")
node_name = form.get("node_name",None)
clustername = form.get("clustername", None)
if node_name is None or clustername is None:
return json.dumps({'success':'false', 'message': 'Illegal form.'})
[status, message] = G_vclustermgr.delete_port_mapping(user,clustername,node_name)
if status is True:
return json.dumps({'success':'true', 'action':'addproxy'})
else:
return json.dumps({'success':'false', 'message': message})
@app.route("/monitor/hosts/<com_id>/<issue>/", methods=['POST'])
@login_required
def hosts_monitor(user, beans, form, com_id, issue):
@ -746,6 +779,9 @@ if __name__ == '__main__':
G_imagemgr = imagemgr.ImageMgr()
logger.info("imagemgr started")
#init portcontrol
portcontrol.init_new()
logger.info("startting to listen on: ")
masterip = env.getenv('MASTER_IP')
logger.info("using MASTER_IP %s", masterip)

View File

@ -1,6 +1,7 @@
#!/usr/bin/python3
import subprocess
import subprocess,env
from log import logger
class ipcontrol(object):
@staticmethod
@ -300,3 +301,82 @@ class netcontrol(object):
ovscontrol.del_port("docklet-br-"+str(uid),port)
ovscontrol.add_port_gre_withkey("docklet-br-"+str(uid), "gre-"+str(uid)+"-"+GatewayHost, GatewayHost, str(uid))
ovscontrol.add_port("docklet-br-"+str(uid), portname)
free_ports = [False]*65536
allocated_ports = {}
class portcontrol(object):
@staticmethod
def init_new():
Free_Ports_str = env.getenv("ALLOCATED_PORTS")
global free_ports
#logger.info(Free_Ports_str)
portsranges=Free_Ports_str.split(',')
#logger.info(postranges)
for portsrange in portsranges:
portsrange=portsrange.strip().split('-')
start = int(portsrange[0])
end = int(portsrange[1])
if end < start or end > 65535 or start < 1:
return [False, "Illegal port ranges."]
i = start
#logger.info(str(start)+" "+str(end))
while i <= end:
free_ports[i] = True
i += 1
#logger.info(free_ports[10001])
return [True,""]
@staticmethod
def init_recovery(Free_Ports_str):
Free_Ports_str = env.getenv("ALLOCATED_PORTS")
return [True,""]
@staticmethod
def acquire_port_mapping(container_name, container_ip, container_port, host_port=None):
global free_ports
global allocated_ports
if container_name in allocated_ports.keys():
return [False, "This container already has a port mapping."]
if container_name == "" or container_ip == "" or container_port == "":
return [False, "Node Name or Node IP or Node Port can't be null."]
#print("acquire_port_mapping1")
free_port = 1
if host_port is not None:
# recover from host_port
free_port = int(host_port)
else:
# acquire new free port
while free_port <= 65535:
if free_ports[free_port]:
break
free_port += 1
if free_port == 65536:
return [False, "No free ports."]
free_ports[free_port] = False
allocated_ports[container_name] = free_port
public_ip = env.getenv("PUBLIC_IP")
try:
subprocess.run(['iptables','-t','nat','-A','PREROUTING','-d',public_ip,'-p','tcp','--dport',str(free_port),"-j","DNAT",'--to-destination','%s:%s'%(container_ip,container_port)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True)
subprocess.run(['iptables','-t','nat','-A','POSTROUTING','-d',container_ip,'-p','tcp','--dport',str(container_port),"-j","SNAT",'--to',public_ip], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True)
return [True, str(free_port)]
except subprocess.CalledProcessError as suberror:
return [False, "set port mapping failed : %s" % suberror.stdout.decode('utf-8')]
@staticmethod
def release_port_mapping(container_name, container_ip, container_port):
global free_ports
global allocated_ports
if container_name not in allocated_ports.keys():
return [False, "This container does not have a port mapping."]
free_port = allocated_ports[container_name]
public_ip = env.getenv("PUBLIC_IP")
try:
subprocess.run(['iptables','-t','nat','-D','PREROUTING','-d',public_ip,'-p','tcp','--dport',str(free_port),"-j","DNAT",'--to-destination','%s:%s'%(container_ip,container_port)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True)
subprocess.run(['iptables','-t','nat','-D','POSTROUTING','-d',container_ip,'-p','tcp','--dport',str(container_port),"-j","SNAT",'--to',public_ip], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, check=True)
except subprocess.CalledProcessError as suberror:
return [False, "release port mapping failed : %s" % suberror.stdout.decode('utf-8')]
free_ports[free_port] = True
allocated_ports.pop(container_name)
return [True, ""]

View File

@ -9,6 +9,7 @@ import env
import proxytool
import requests
import traceback
from nettools import portcontrol
userpoint = "http://" + env.getenv('USER_IP') + ":" + str(env.getenv('USER_PORT'))
def post_to_user(url = '/', data={}):
@ -160,6 +161,7 @@ class VclusterMgr(object):
info['proxy_url'] = proxy_url
info['proxy_server_ip'] = proxy_server_ip
info['proxy_public_ip'] = proxy_public_ip
info['port_mapping'] = []
clusterfile.write(json.dumps(info))
clusterfile.close()
return [True, info]
@ -245,6 +247,60 @@ class VclusterMgr(object):
clusterfile.close()
return [True, clusterinfo]
def add_port_mapping(self,username,clustername,node_name,node_ip, port):
[status, clusterinfo] = self.get_clusterinfo(clustername, username)
host_port = 0
if self.distributedgw == 'True':
worker = self.nodemgr.ip_to_rpc(clusterinfo['proxy_server_ip'])
[success, host_port] = worker.acquire_port_mapping(node_name, node_ip, port)
else:
[success, host_port] = portcontrol.acquire_port_mapping(node_name, node_ip, port)
if not success:
return [False, host_port]
if 'port_mapping' not in clusterinfo.keys():
clusterinfo['port_mapping'] = []
clusterinfo['port_mapping'].append({'node_name':node_name, 'node_ip':node_ip, 'node_port':port, 'host_port':host_port})
clusterfile = open(self.fspath + "/global/users/" + username + "/clusters/" + clustername, 'w')
clusterfile.write(json.dumps(clusterinfo))
clusterfile.close()
return [True, clusterinfo]
def recover_port_mapping(self,username,clustername):
[status, clusterinfo] = self.get_clusterinfo(clustername, username)
for rec in clusterinfo['port_mapping']:
if self.distributedgw == 'True':
worker = self.nodemgr.ip_to_rpc(clusterinfo['proxy_server_ip'])
[success, host_port] = worker.acquire_port_mapping(rec['node_name'], rec['node_ip'], rec['node_port'], rec['host_port'])
else:
[success, host_port] = portcontrol.acquire_port_mapping(rec['node_name'], rec['node_ip'], rec['node_port'], rec['host_port'])
if not success:
return [False, host_port]
return [True, clusterinfo]
def delete_port_mapping(self, username, clustername, node_name):
[status, clusterinfo] = self.get_clusterinfo(clustername, username)
idx = 0
for item in clusterinfo['port_mapping']:
if item['node_name'] == node_name:
break
idx += 1
if idx == len(clusterinfo['port_mapping']):
return [False,"No port mapping."]
node_ip = clusterinfo['port_mapping'][idx]['node_ip']
node_port = clusterinfo['port_mapping'][idx]['node_port']
if self.distributedgw == 'True':
worker = self.nodemgr.ip_to_rpc(clusterinfo['proxy_server_ip'])
[success,msg] = worker.release_port_mapping(node_name, node_ip, node_port)
else:
[success,msg] = portcontrol.release_port_mapping(node_name, node_ip, node_port)
if not success:
return [False,msg]
clusterinfo['port_mapping'].pop(idx)
clusterfile = open(self.fspath + "/global/users/" + username + "/clusters/" + clustername, 'w')
clusterfile.write(json.dumps(clusterinfo))
clusterfile.close()
return [True, clusterinfo]
def flush_cluster(self,username,clustername,containername):
begintime = datetime.datetime.now()
[status, info] = self.get_clusterinfo(clustername, username)
@ -381,6 +437,10 @@ class VclusterMgr(object):
new_hostinfo.append(host)
hostfile.writelines(new_hostinfo)
hostfile.close()
[success, msg] = self.delete_port_mapping(username, clustername, containername)
if not success:
return [False, msg]
[status, info] = self.get_clusterinfo(clustername, username)
return [True, info]
def get_clustersetting(self, clustername, username, containername, allcontainer):
@ -508,6 +568,9 @@ class VclusterMgr(object):
self.update_proxy_ipAndurl(clustername,username,info['proxy_server_ip'])
[status, info] = self.get_clusterinfo(clustername, username)
self.update_cluster_baseurl(clustername,username,info['proxy_server_ip'],info['proxy_public_ip'])
if not 'port_mapping' in info.keys():
info['port_mapping'] = []
self.write_clusterinfo(info,clustername,username)
if info['status'] == 'stopped':
return [True, "cluster no need to start"]
# recover proxy of cluster
@ -545,6 +608,10 @@ class VclusterMgr(object):
namesplit = container['containername'].split('-')
portname = namesplit[1] + '-' + namesplit[2]
worker.recover_usernet(portname, uid, info['proxy_server_ip'], container['host']==info['proxy_server_ip'])
# recover ports mapping
[success, msg] = self.recover_port_mapping(username,clustername)
if not success:
return [False, msg]
return [True, "start cluster"]
# maybe here should use cluster id
@ -560,10 +627,12 @@ class VclusterMgr(object):
else:
proxytool.delete_route("/" + info['proxy_public_ip'] + '/go/'+username+'/'+clustername)
for container in info['containers']:
self.delete_port_mapping(username,clustername,container['containername'])
worker = xmlrpc.client.ServerProxy("http://%s:%s" % (container['host'], env.getenv("WORKER_PORT")))
if worker is None:
return [False, "The worker can't be found or has been stopped."]
worker.stop_container(container['containername'])
[status, info] = self.get_clusterinfo(clustername, username)
info['status']='stopped'
info['start_time']="------"
infofile = open(self.fspath+"/global/users/"+username+"/clusters/"+clustername, 'w')

View File

@ -14,7 +14,7 @@ import xmlrpc.server, sys, time
from socketserver import ThreadingMixIn
import threading
import etcdlib, network, container
from nettools import netcontrol,ovscontrol
from nettools import netcontrol,ovscontrol,portcontrol
import monitor, proxytool
from lvmtool import new_group, recover_group
@ -115,6 +115,10 @@ class Worker(object):
else:
logger.error ("worker init mode:%s not supported" % value)
sys.exit(1)
# init portcontrol
logger.info("init portcontrol")
portcontrol.init_new()
# initialize rpc
# xmlrpc.server.SimpleXMLRPCServer(addr) -- addr : (ip-addr, port)
# if ip-addr is "", it will listen ports of all IPs of this host
@ -133,6 +137,8 @@ class Worker(object):
self.rpcserver.register_function(netcontrol.recover_usernet)
self.rpcserver.register_function(proxytool.set_route)
self.rpcserver.register_function(proxytool.delete_route)
self.rpcserver.register_function(portcontrol.acquire_port_mapping)
self.rpcserver.register_function(portcontrol.release_port_mapping)
# register functions or instances to server for rpc
#self.rpcserver.register_function(function_name)

View File

@ -44,7 +44,7 @@
<button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
</div>
</div>
<div class="box-body" style="display:none">
<div class="box-body" style="display:none">
<div class="row">
<div class="col-md-12">
<div class="box box-info">
@ -210,7 +210,6 @@
</div>
</div>
</div>
</div>
</tr>
{% endfor %}
@ -218,38 +217,90 @@
</table>
</div>
</div>
</div>
</div>
</div>
<br/>
<div class="row">
<div class="col-md-12">
<div class="row">
<div class="col-md-12">
<div class="box box-info">
<div class="box-header with-border">
<h3 class="box-title">SERVICE</h3>
<h5><a href="{{ clusterinfo['proxy_url'] }}" title="click here jump to your proxy server">{{ clusterinfo['proxy_url'] }}</a></h5>
<h4 class="box-title">TCP Ports Mapping</h4>
<div class="box-tools pull-right">
<button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
</button>
<button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
</div>
</div>
<div class="box-body">
<form action="/addproxy/{{master.split("@")[0]}}/{{ clustername }}/" id="addproxy" method="POST">
{% if 'proxy_ip' in clusterinfo %}
<p>ip:<input type="text" id="proxy_ip" name="proxy_ip" value={{ clusterinfo['proxy_ip'][:clusterinfo['proxy_ip'].index(':')] }} readonly="true"/>port:<input type="text" id="proxy_port" name="proxy_port" value={{ clusterinfo['proxy_ip'][clusterinfo['proxy_ip'].index(':')+1:] }} readonly="true"/>
<button type="button" class="btn-xs btn-default">enable</button>
<a href="/deleteproxy/{{master.split("@")[0]}}/{{ clustername }}/"><button type="button" class="btn-xs btn-danger">disable</button></a></p>
{% else %}
<p>ip:<input type="text" id="proxy_ip" name="proxy_ip" value={{ clusterinfo["containers"][0]["ip"][:clusterinfo["containers"][0]["ip"].index("/")] }} />port:<input type="text" id="proxy_port" name="proxy_port" value="80"/>
<button type="submit" class="btn-xs btn-success">enable</button>
<button type="button" class="btn-xs btn-default">disable</button></p>
{% endif %}
</form>
</div>
</div>
</div>
</div>
<div class="box-body">
<p>
<button type="button" class="btn btn-primary btn-sm" data-toggle="modal" data-target="#Addportsmapping_{{ clustername }}_{{master.split("@")[1]}}"><i class="fa fa-plus"></i>Apply</button>
</p>
<div class="modal inmodal" id="Addportsmapping_{{ clustername }}_{{master.split("@")[1]}}" tabindex="-1" role="dialog" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content animated fadeIn">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
<i class="fa fa-plus modal-icon"></i>
<h4 class="modal-title">Apply for TCP Ports Mapping</h4>
<small class="font-bold">Add a TCP port mapping for a node</small>
</div>
<div class="modal-body">
<form action="/port_mapping/add/{{master.split("@")[0]}}/" method="POST" id="AddportsmappingForm">
<div class="form-group">
<label>Cluster Name</label>
<input type = "text" value="{{ clustername }}" class="form-control" name="clustername" readonly="readonly">
</div>
<div class="form-group">
<label>Node Name</label>
<select class="form-control" name="node_name" onchange="chnodeip(this.value,node_ip)">
{% for container in clusterinfo['containers'] %}
<option value="{{ container['containername'] }}">{{ container['containername'] }}</option>
{% endfor %}
</select>
</div>
<div class="form-group">
<label>Node IP</label>
<input type = "text" value="{{ clusterinfo["containers"][0]["ip"][:clusterinfo["containers"][0]["ip"].index("/")] }}" class="form-control" name="node_ip" readonly="readonly">
</div>
<div class="form-group">
<label>Node Port</label><small class="font-bold"> The port that the host port is mapping to(1-65535).</small>
<input type="number" class="form-control" placeholder="1-65535" value="80" name="node_port" id="node_port" min="1" max="65535"/>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-white" data-dismiss="modal">Close</button>
<button type="submit" class="btn btn-primary">Submit</button>
</div>
</form>
</div>
</div>
</div>
</div>
<table class="table table-bordered">
<thead>
<tr>
<th>Node Name</th>
<th>Node IP</th>
<th>Node Port</th>
<th>Host Public IP</th>
<th>Host Port</th>
<th>Delete</th>
</tr>
</thead>
<tbody>
{% for record in clusterinfo['port_mapping'] %}
<tr>
<td>{{ record['node_name'] }}</td>
<td>{{ record['node_ip'] }}</td>
<td>{{ record['node_port'] }}</td>
<td>{{ clusterinfo['proxy_public_ip'] }}</td>
<td>{{ record['host_port'] }}</td>
<td><a class="btn btn-xs btn-danger" href="/port_mapping/delete/{{master.split("@")[0]}}/{{ clustername }}/{{ record['node_name'] }}/">Delete</a></td>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
@ -356,7 +407,18 @@
$(".table-image").DataTable();
$(".table-image").attr("style","");
});
var map_node_ip = [];
{% for master in allclusters %}
{% for clustername, clusterinfo in allclusters[master].items() %}
{% for container in clusterinfo['containers'] %}
map_node_ip["{{ container['containername'] }}"] = "{{ container["ip"][:container["ip"].index("/")] }}";
{% endfor %}
{% endfor %}
{% endfor %}
function chnodeip(node_name,field)
{
field.value = map_node_ip[node_name];
}
</script>
{% endblock %}

View File

@ -212,21 +212,35 @@ def saveImage_force(clustername,containername,masterip):
saveImageView.description = request.form['description']
return saveImageView.as_view()
@app.route("/addproxy/<masterip>/<clustername>/", methods=['POST'])
'''@app.route("/addproxy/<masterip>/<clustername>/", methods=['POST'])
@login_required
def addproxy(clustername,masterip):
addproxyView.clustername = clustername
addproxyView.masterip = masterip
addproxyView.ip = request.form['proxy_ip']
addproxyView.port = request.form['proxy_port']
return addproxyView.as_view()
return addproxyView.as_view()'''
@app.route("/deleteproxy/<masterip>/<clustername>/", methods=['GET'])
'''@app.route("/deleteproxy/<masterip>/<clustername>/", methods=['GET'])
@login_required
def deleteproxy(clustername,masterip):
deleteproxyView.clustername = clustername
deleteproxyView.masterip = masterip
return deleteproxyView.as_view()
return deleteproxyView.as_view()'''
@app.route("/port_mapping/add/<masterip>/", methods=['POST'])
@login_required
def addPortMapping(masterip):
addPortMappingView.masterip = masterip
return addPortMappingView.as_view()
@app.route("/port_mapping/delete/<masterip>/<clustername>/<node_name>/", methods=['GET'])
@login_required
def delPortMapping(masterip,clustername,node_name):
delPortMappingView.masterip = masterip
delPortMappingView.clustername = clustername
delPortMappingView.node_name = node_name
return delPortMappingView.as_view()
@app.route("/getmasterdesc/<mastername>/", methods=['POST'])
@login_required

View File

@ -76,7 +76,7 @@ class createClusterView(normalView):
class descriptionMasterView(normalView):
template_path = "description.html"
@classmethod
def get(self):
return self.render(self.template_path, description=self.desc)
@ -403,3 +403,37 @@ class configView(normalView):
@classmethod
def post(self):
return self.get()
class addPortMappingView(normalView):
template_path = "error.html"
@classmethod
def post(self):
data = {"clustername":request.form["clustername"],"node_name":request.form["node_name"],"node_ip":request.form["node_ip"],"node_port":request.form["node_port"]}
result = dockletRequest.post('/port_mapping/add/',data, self.masterip)
success = result.get("success")
if success == "true":
return redirect("/config/")
else:
return self.render(self.template_path, message = result.get("message"))
@classmethod
def get(self):
return self.post()
class delPortMappingView(normalView):
template_path = "error.html"
@classmethod
def post(self):
data = {"clustername":self.clustername,"node_name":self.node_name}
result = dockletRequest.post('/port_mapping/delete/',data, self.masterip)
success = result.get("success")
if success == "true":
return redirect("/config/")
else:
return self.render(self.template_path, message = result.get("message"))
@classmethod
def get(self):
return self.post()