m26508134
/
docklet
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #355 from FirmlyReality/batch 

Batch
This commit is contained in:
Yujian Zhu 2018-11-19 00:41:00 +08:00 committed by GitHub
parent b3a3673b50 2007f068e5
commit 883814f0da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 174 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/master/bugreporter.py Normal file
View File

@ -0,0 +1,53 @@
from master.settings import settings
import smtplib
from utils.log import logger
from utils import env
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.header import Header
from datetime import datetime
import json
def send_bug_mail(username, bugmessage):
#admin_email_address = env.getenv('ADMIN_EMAIL_ADDRESS')
nulladdr = ['\'\'', '\"\"', '']
email_from_address = settings.get('EMAIL_FROM_ADDRESS')
admin_email_address = settings.get('ADMIN_EMAIL_ADDRESS')
logger.info("receive bug from %s: %s" % (username, bugmessage))
if (email_from_address in nulladdr or admin_email_address in nulladdr):
return {'success': 'false'}
#text = 'Dear '+ username + ':\n' + ' Your account in docklet has been activated'
text = '<html><h4>Dear '+ 'admin' + ':</h4>'
text += '''<p>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;A bug has been report by %s.</p>
<br/>
<strong>&nbsp; %s &nbsp;</strong>
<br/>
<p>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Please check it !</p>
<br/><br/>
<p> Docklet Team, SEI, PKU</p>
''' % (username, bugmessage)
text += '<p>'+ str(datetime.utcnow()) + '</p>'
text += '</html>'
subject = 'A bug of Docklet has been reported'
if admin_email_address[0] == '"':
admins_addr = admin_email_address[1:-1].split(" ")
else:
admins_addr = admin_email_address.split(" ")
alladdr=""
for addr in admins_addr:
alladdr = alladdr+addr+", "
alladdr=alladdr[:-2]
msg = MIMEMultipart()
textmsg = MIMEText(text,'html','utf-8')
msg['Subject'] = Header(subject, 'utf-8')
msg['From'] = email_from_address
msg['To'] = alladdr
msg.attach(textmsg)
s = smtplib.SMTP()
s.connect()
try:
s.sendmail(email_from_address, admins_addr, msg.as_string())
except Exception as e:
logger.error(e)
s.close()
return {'success':'true'}

4
src/master/vclustermgr.py
View File

@ -116,6 +116,8 @@ class VclusterMgr(object):
def create_cluster(self, clustername, username, image, user_info, setting): def create_cluster(self, clustername, username, image, user_info, setting):
if self.is_cluster(clustername, username): if self.is_cluster(clustername, username):
return [False, "cluster:%s already exists" % clustername] return [False, "cluster:%s already exists" % clustername]
if self.imgmgr.get_image_size(image) + 100 > int(setting["disk"]):
return [False, "the size of disk is not big enough for the image"]
clustersize = int(self.defaultsize) clustersize = int(self.defaultsize)
logger.info ("starting cluster %s with %d containers for %s" % (clustername, int(clustersize), username)) logger.info ("starting cluster %s with %d containers for %s" % (clustername, int(clustersize), username))
workers = self.nodemgr.get_base_nodeips() workers = self.nodemgr.get_base_nodeips()
@ -202,6 +204,8 @@ class VclusterMgr(object):
def scale_out_cluster(self,clustername,username, image,user_info, setting): def scale_out_cluster(self,clustername,username, image,user_info, setting):
if not self.is_cluster(clustername,username): if not self.is_cluster(clustername,username):
return [False, "cluster:%s not found" % clustername] return [False, "cluster:%s not found" % clustername]
if self.imgmgr.get_image_size(image) + 100 > int(setting["disk"]):
return [False, "the size of disk is not big enough for the image"]
workers = self.nodemgr.get_base_nodeips() workers = self.nodemgr.get_base_nodeips()
if (len(workers) == 0): if (len(workers) == 0):
logger.warning("no workers to start containers, scale out failed") logger.warning("no workers to start containers, scale out failed")

29
src/utils/imagemgr.py
View File

@ -380,6 +380,29 @@ class ImageMgr():
return "" return ""
return image.description return image.description
def get_image_size(self, image):
imagename = image['name']
imagetype = image['type']
imageowner = image['owner']
if imagename == "base" and imagetype == "base":
return 0
if imagetype == "private":
imgpath = self.imgpath + "private/" + user + "/"
else:
imgpath = self.imgpath + "public/" + imageowner + "/"
return os.stat(os.path.join(imgpath, imagename)).st_size // (1024*1024)
def format_size(self, size_in_byte):
if size_in_byte < 1024:
return str(size_in_byte) + "B"
elif size_in_byte < 1024*1024:
return str(size_in_byte//1024) + "KB"
elif size_in_byte < 1024*1024*1024:
return str(size_in_byte//(1024*1024)) + "MB"
else:
return str(size_in_byte//(1024*1024*1024)) + "GB"
def list_images(self,user): def list_images(self,user):
images = {} images = {}
images["private"] = [] images["private"] = []
@ -398,6 +421,9 @@ class ImageMgr():
[time, description] = self.get_image_info(user, imagename, "private") [time, description] = self.get_image_info(user, imagename, "private")
fimage["time"] = time fimage["time"] = time
fimage["description"] = description fimage["description"] = description
fimage["size"] = os.stat(os.path.join(imgpath, image)).st_size
fimage["size_format"] = self.format_size(fimage["size"])
fimage["size_in_mb"] = fimage["size"] // (1024*1024)
images["private"].append(fimage) images["private"].append(fimage)
except Exception as e: except Exception as e:
logger.error(e) logger.error(e)
@ -423,6 +449,9 @@ class ImageMgr():
[time, description] = self.get_image_info(public_user, imagename, "public") [time, description] = self.get_image_info(public_user, imagename, "public")
fimage["time"] = time fimage["time"] = time
fimage["description"] = description fimage["description"] = description
fimage["size"] = os.stat(os.path.join(imgpath, image)).st_size
fimage["size_format"] = self.format_size(fimage["size"])
fimage["size_in_mb"] = fimage["size"] // (1024*1024)
images["public"][public_user].append(fimage) images["public"][public_user].append(fimage)
except Exception as e: except Exception as e:
logger.error(e) logger.error(e)

8
user/user.py
View File

@ -34,6 +34,7 @@ from utils.model import User,db
from httplib2 import Http from httplib2 import Http
from urllib.parse import urlencode from urllib.parse import urlencode
from master.settings import settings from master.settings import settings
from master.bugreporter import send_bug_mail
external_login = env.getenv('EXTERNAL_LOGIN') external_login = env.getenv('EXTERNAL_LOGIN')
if(external_login == 'TRUE'): if(external_login == 'TRUE'):
@ -472,6 +473,13 @@ def query_self_notifications_infos(cur_user, user, form):
result = G_notificationmgr.query_self_notifications_infos(cur_user=cur_user, form=form) result = G_notificationmgr.query_self_notifications_infos(cur_user=cur_user, form=form)
return json.dumps(result) return json.dumps(result)
@app.route("/bug/report/", methods=['POST'])
@login_required
def report_bug(cur_user, user, form):
logger.info("handle request: bug/report")
result = send_bug_mail(user, form.get("bugmessage", None))
return json.dumps(result)
@app.route("/billing/beans/", methods=['POST']) @app.route("/billing/beans/", methods=['POST'])
@auth_key_required @auth_key_required
def billing_beans(): def billing_beans():

22
web/templates/addCluster.html
View File

@ -64,6 +64,7 @@
<th>ImageName</th> <th>ImageName</th>
<th>Type</th> <th>Type</th>
<th>Owner</th> <th>Owner</th>
<th>Size</th>
<th>Description</th> <th>Description</th>
<th>Choose</th> <th>Choose</th>
</tr> </tr>
@ -73,16 +74,18 @@
<td>base</td> <td>base</td>
<td>public</td> <td>public</td>
<td>docklet</td> <td>docklet</td>
<td>--</td>
<td>A base image for you</td> <td>A base image for you</td>
<td><div class="i-checks"><label><input type="radio" name="image" value="base_base_base" checked="checked"></label></div></td> <td><div class="i-checks"><label><input type="radio" name="image" value="base_base_base" checked="checked" onchange="setMinDiskSize(0)"></label></div></td>
</tr> </tr>
{% for image in images['private'] %} {% for image in images['private'] %}
<tr> <tr>
<td>{{image['name']}}</td> <td>{{image['name']}}</td>
<td>private</td> <td>private</td>
<td>{{user}}</td> <td>{{user}}</td>
<td><a href="/image/{{masterips[0].split("@")[1]}}/description/{{image['name']}}_{{user}}_private/" target="_blank">{{image['description']}}</a></td> <td>{{image['size_format']}}</td>
<td><div class="i-checks"><label><input type="radio" name="image" value="{{image['name']}}_{{user}}_private"></label></div></td> <td><a href="/image/{{masterips[0].split("@")[0]}}/description/{{image['name']}}_{{user}}_private/" target="_blank">{{image['description']}}</a></td>
<td><div class="i-checks"><label><input type="radio" name="image" value="{{image['name']}}_{{user}}_private" onchange="setMinDiskSize({{image['size_in_mb']}})"></label></div></td>
</tr> </tr>
{% endfor %} {% endfor %}
{% for p_user,p_images in images['public'].items() %} {% for p_user,p_images in images['public'].items() %}
@ -91,8 +94,9 @@
<td>{{image['name']}}</td> <td>{{image['name']}}</td>
<td>public</td> <td>public</td>
<td>{{p_user}}</td> <td>{{p_user}}</td>
<td><a href="/image/{{masterips[0].split("@")[1]}}/description/{{image['name']}}_{{p_user}}_public/" target="_blank">{{image['description']}}</a></td> <td>{{image['size_format']}}</td>
<td><div class="i-checks"><label><input type="radio" name="image" value="{{image['name']}}_{{p_user}}_public"></label></div></td> <td><a href="/image/{{masterips[0].split("@")[0]}}/description/{{image['name']}}_{{p_user}}_public/" target="_blank">{{image['description']}}</a></td>
<td><div class="i-checks"><label><input type="radio" name="image" value="{{image['name']}}_{{p_user}}_public" onchange="setMinDiskSize({{image['size_in_mb']}})"></label></div></td>
</tr> </tr>
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
@ -125,7 +129,7 @@
</div> </div>
<div class="form-group"> <div class="form-group">
<label class="col-sm-2 control-label">DISK</label> <label class="col-sm-2 control-label">DISK</label>
<div class="col-sm-3"><input type="number" class="form-control" name="diskSetting" id="diskSetting" value= {{defaultsetting['disk']}} /> {{usage['disk']}} MB/{{quota['disk']}}MB <div class="col-sm-3"><input type="number" class="form-control" name="diskSetting" id="diskSetting" value= {{defaultsetting['disk']}} /> {{usage['disk']}} MB/{{quota['disk']}}MB (min value is the size of image + 100)
</div> </div>
</div> </div>
</div> </div>
@ -167,6 +171,12 @@
<script src="http://cdn.bootcss.com/datatables/1.10.11/js/dataTables.bootstrap.js"></script> <script src="http://cdn.bootcss.com/datatables/1.10.11/js/dataTables.bootstrap.js"></script>
<script src="http://cdn.bootcss.com/datatables-tabletools/2.1.5/js/TableTools.min.js"></script> <script src="http://cdn.bootcss.com/datatables-tabletools/2.1.5/js/TableTools.min.js"></script>
<script type="text/javascript">
function setMinDiskSize(minSize) {
var disk = document.getElementById('diskSetting');
disk.setAttribute('min', minSize+100);
}
</script>
<script type="text/javascript"> <script type="text/javascript">
$("select#masterselector").change(function() { $("select#masterselector").change(function() {

21
web/templates/base_AdminLTE.html
View File

@ -109,6 +109,23 @@
</ul> </ul>
</li> </li>
<!-- Control Sidebar Toggle Button --> <!-- Control Sidebar Toggle Button -->
<li class="dropdown user user-menu">
<!-- Menu Toggle Button -->
<a href="#" class="dropdown-toggle" data-toggle="dropdown">
<strong>Bug Report</strong>
</a>
<ul class="dropdown-menu">
<!-- The user image in the menu -->
<form action="/bug/report/" method="POST">
<textarea id="bugmessage" name="bugmessage" style="width:250px; height:200px"></textarea>
<li class="user-footer" style="background-color:#e6e6e6">
<div class="pull-right">
<button type="submit" class="btn btn-default btn-flat">Submit</a>
</div>
</li>
</form>
</ul>
</li>
<li> <li>
<a href="/document/" target="_blank"><strong>Help</strong></a> <a href="/document/" target="_blank"><strong>Help</strong></a>
</li> </li>
@ -223,10 +240,10 @@
<footer class="main-footer"> <footer class="main-footer">
<!-- To the right --> <!-- To the right -->
<div class="pull-right hidden-xs"> <div class="pull-right hidden-xs">
<i><a href="http://github.com/unias/docklet">Docklet {{ version }}</a></i> <i><a href="https://github.com/unias/docklet">Docklet {{ version }}</a></i>
</div> </div>
<!-- Default to the left --> <!-- Default to the left -->
<strong>Copyright</strong>&copy;&nbsp;2017 <a href="http://unias.github.io/docklet">UniAS</a>@<a href="http://www.sei.pku.edu.cn"> SEI, PKU</a> <strong>Copyright</strong>&copy;&nbsp;2017 <a href="https://unias.github.io/docklet">UniAS</a>@<a href="http://www.sei.pku.edu.cn"> SEI, PKU</a>
</footer> </footer>

10
web/templates/config.html
View File

@ -80,6 +80,7 @@
<th>ImageName</th> <th>ImageName</th>
<th>Type</th> <th>Type</th>
<th>Owner</th> <th>Owner</th>
<th>Size</th>
<th>Choose</th> <th>Choose</th>
</tr> </tr>
</thead> </thead>
@ -88,6 +89,7 @@
<td>base</td> <td>base</td>
<td>public</td> <td>public</td>
<td>docklet</td> <td>docklet</td>
<td>--</td>
<td><input type="radio" name="image" value="base_base_base" checked="checked"></td> <td><input type="radio" name="image" value="base_base_base" checked="checked"></td>
</tr> </tr>
{% for image in allimages[master]['private'] %} {% for image in allimages[master]['private'] %}
@ -95,6 +97,7 @@
<td>{{image['name']}}</td> <td>{{image['name']}}</td>
<td>private</td> <td>private</td>
<td>{{mysession['username']}}</td> <td>{{mysession['username']}}</td>
<td>{{image['size_format']}}</td>
<td><input type="radio" name="image" value="{{image['name']}}_{{mysession['username']}}_private"></td> <td><input type="radio" name="image" value="{{image['name']}}_{{mysession['username']}}_private"></td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -104,6 +107,7 @@
<td>{{image['name']}}</td> <td>{{image['name']}}</td>
<td>public</td> <td>public</td>
<td>{{p_user}}</td> <td>{{p_user}}</td>
<td>{{image['size_format']}}</td>
<td><input type="radio" name="image" value="{{image['name']}}_{{p_user}}_public"></td> <td><input type="radio" name="image" value="{{image['name']}}_{{p_user}}_public"></td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -134,7 +138,7 @@
</div> </div>
<div class="form-group"> <div class="form-group">
<label class="control-label">DISK</label> <label class="control-label">DISK</label>
<div><input type="number" class="form-control" name="diskSetting" id="diskSetting" value= {{defaultsetting['disk']}} /> {{usage['disk']}} MB/{{quota['disk']}}MB <div><input type="number" class="form-control" name="diskSetting" id="diskSetting" value= {{defaultsetting['disk']}} /> {{usage['disk']}} MB/{{quota['disk']}}MB (min value is the size of image + 100)
</div> </div>
</div> </div>
</div> </div>
@ -327,6 +331,7 @@
<th>Type</th> <th>Type</th>
<th>Owner</th> <th>Owner</th>
<th>CreateTime</th> <th>CreateTime</th>
<th>Size</th>
<th>Description</th> <th>Description</th>
<th>Location</th> <th>Location</th>
<th>Status</th> <th>Status</th>
@ -339,6 +344,7 @@
<td>public</td> <td>public</td>
<td>docklet</td> <td>docklet</td>
<td>2015-01-01 00:00:00</td> <td>2015-01-01 00:00:00</td>
<td>--</td>
<td>A Base Image For You</td> <td>A Base Image For You</td>
<td>--</td> <td>--</td>
<td></td> <td></td>
@ -351,6 +357,7 @@
<td>private</td> <td>private</td>
<td>{{mysession['username']}}</td> <td>{{mysession['username']}}</td>
<td>{{image['time']}}</td> <td>{{image['time']}}</td>
<td>{{image['size_format']}}</td>
<td><a href="/image/{{master.split("@")[0]}}/description/{{image['name']}}_{{mysession['username']}}_private/" target="_blank">{{image['description']}}</a></td> <td><a href="/image/{{master.split("@")[0]}}/description/{{image['name']}}_{{mysession['username']}}_private/" target="_blank">{{image['description']}}</a></td>
<td>{{master.split("@")[1]}}</td> <td>{{master.split("@")[1]}}</td>
{% if image['isshared'] == 'false' %} {% if image['isshared'] == 'false' %}
@ -414,6 +421,7 @@
<td>public</td> <td>public</td>
<td>{{p_user}}</td> <td>{{p_user}}</td>
<td>{{image['time']}}</td> <td>{{image['time']}}</td>
<td>{{image['size_format']}}</td>
<td><a href="/image/{{master.split("@")[0]}}/description/{{image['name']}}_{{p_user}}_public/" target="_blank">{{image['description']}}</a></td> <td><a href="/image/{{master.split("@")[0]}}/description/{{image['name']}}_{{p_user}}_public/" target="_blank">{{image['description']}}</a></td>
<td>{{master.split("@")[1]}}</td> <td>{{master.split("@")[1]}}</td>
<td></td> <td></td>

4
web/templates/home.template
View File

@ -38,12 +38,12 @@
<div class="container"> <div class="container">
<div class="row" style="font-size:16px; color:white; padding:16px"> <div class="row" style="font-size:16px; color:white; padding:16px">
<div class="pull-right" > <div class="pull-right" >
<a class="linkbtn" href="http://docklet.unias.org/docklet-book/userguide/_book/">Document</a> <a class="linkbtn" href="https://unias.github.io/docklet/userguide/">Document</a>
&centerdot; &centerdot;
<a class="linkbtn" href="/login/" >Sign In</a> <a class="linkbtn" href="/login/" >Sign In</a>
</div> </div>
<div> <div>
<a class="linkbtn" href="http://docklet.unias.org"><strong>Docklet Cloud OS</strong></a> <a class="linkbtn" href="https://docklet.unias.org"><strong>Docklet Cloud OS</strong></a>
</div> </div>
</div> </div>
</div> </div>

13
web/templates/monitor/history.html
View File

@ -32,8 +32,8 @@
</div> </div>
</div> </div>
<div class="box-body"> <div class="box-body">
<div class="table table-responsive"> <div class="table">
<table class="table table-striped table-bordered table-hover table-image" > <table width="100%" cellspacing="0" style="margin:0 auto;" class="table table-striped table-bordered table-hover table-history" >
<thead> <thead>
<tr> <tr>
<th>NO</th> <th>NO</th>
@ -65,17 +65,14 @@
{% endblock %} {% endblock %}
{% block script_src %} {% block script_src %}
<script src="//cdn.bootcss.com/jquery/2.2.1/jquery.min.js"></script> <script src="//cdn.bootcss.com/datatables/1.10.11/js/jquery.dataTables.min.js"></script>
<script src="http://cdn.bootcss.com/datatables/1.10.11/js/jquery.dataTables.js"></script> <script src="//cdn.bootcss.com/datatables/1.10.11/js/dataTables.bootstrap.min.js"></script>
<script src="http://cdn.bootcss.com/datatables/1.10.11/js/dataTables.bootstrap.js"></script>
<script src="http://cdn.bootcss.com/datatables-tabletools/2.1.5/js/TableTools.min.js"></script>
<script> <script>
$(document).ready(function() { $(document).ready(function() {
$(".table-image").DataTable(); $(".table-history").DataTable({"scrollX":true});
}); });
</script> </script>
{% endblock %} {% endblock %}

3
web/templates/opsuccess.html
View File

@ -19,6 +19,9 @@
<h1>SUCCESS</h1> <h1>SUCCESS</h1>
<br/> <br/>
<br/> <br/>
<pre>{{message}}</pre>
<br/>
<br/>
<a href="/dashboard/"><span class="btn btn-w-m btn-success">Click Here Back To The Workspace</span></a> <a href="/dashboard/"><span class="btn btn-w-m btn-success">Click Here Back To The Workspace</span></a>
</div> </div>
{% endblock %} {% endblock %}

10
web/web.py
View File

@ -36,6 +36,7 @@ from webViews.admin import *
from webViews.monitor import * from webViews.monitor import *
from webViews.beansapplication import * from webViews.beansapplication import *
from webViews.cloud import * from webViews.cloud import *
from webViews.reportbug import *
from webViews.authenticate.auth import login_required, administration_required,activated_required from webViews.authenticate.auth import login_required, administration_required,activated_required
from webViews.authenticate.register import registerView from webViews.authenticate.register import registerView
from webViews.authenticate.login import loginView, logoutView from webViews.authenticate.login import loginView, logoutView
@ -113,13 +114,19 @@ def dashboard():
@app.route("/document/", methods=['GET']) @app.route("/document/", methods=['GET'])
def redirect_dochome(): def redirect_dochome():
return redirect("http://unias.github.io/docklet/userguide") return redirect("https://unias.github.io/docklet/userguide/")
@app.route("/config/", methods=['GET']) @app.route("/config/", methods=['GET'])
@login_required @login_required
def config(): def config():
return configView.as_view() return configView.as_view()
@app.route("/bug/report/", methods=['POST'])
@login_required
def reportBug():
reportBugView.bugmessage = request.form['bugmessage']
return reportBugView.as_view()
@app.route("/batch_jobs/", methods=['GET']) @app.route("/batch_jobs/", methods=['GET'])
@login_required @login_required
def batch_job(): def batch_job():
@ -632,7 +639,6 @@ def adminpage():
def updatesettings(): def updatesettings():
return updatesettingsView.as_view() return updatesettingsView.as_view()
@app.route('/index/', methods=['GET']) @app.route('/index/', methods=['GET'])
def jupyter_control(): def jupyter_control():
return redirect('/dashboard/') return redirect('/dashboard/')

3
web/webViews/dockletrequest.py
View File

@ -38,7 +38,8 @@ class dockletRequest():
'user', 'user',
'beans', 'beans',
'notification', 'notification',
'settings' 'settings',
'bug'
} }
if ":" not in endpoint: if ":" not in endpoint:
endpoint = "http://"+endpoint+":"+master_port endpoint = "http://"+endpoint+":"+master_port

16
web/webViews/reportbug.py Normal file
View File

@ -0,0 +1,16 @@
from flask import session,render_template,request,redirect
from webViews.view import normalView
from webViews.dockletrequest import dockletRequest
class reportBugView(normalView):
template_path = "opsuccess.html"
@classmethod
def get(self):
dockletRequest.post("/bug/report/", {'bugmessage': self.bugmessage})
return self.render(self.template_path, message="Thank You!")
@classmethod
def post(self):
return self.get()