diff --git a/src/monitor.py b/src/monitor.py index 3e5b275..f8a364b 100755 --- a/src/monitor.py +++ b/src/monitor.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import subprocess,re,os,etcdlib,psutil +import subprocess,re,os,etcdlib,psutil,math import time,threading,json,traceback,platform from log import logger @@ -11,12 +11,16 @@ monitor_vnodes = {} workerinfo = {} workercinfo = {} +lastbillingtime = {} +increment = {} + class Container_Collector(threading.Thread): def __init__(self,test=False): threading.Thread.__init__(self) self.thread_stop = False self.interval = 2 + self.billingtime = 3600 self.test = test self.cpu_last = {} self.cpu_quota = {} @@ -46,6 +50,8 @@ class Container_Collector(threading.Thread): def collect_containerinfo(self,container_name): global workercinfo + global increment + global lastbillingtime output = subprocess.check_output("sudo lxc-info -n %s" % (container_name),shell=True) output = output.decode('utf-8') parts = re.split('\n',output) @@ -54,6 +60,10 @@ class Container_Collector(threading.Thread): basic_exist = 'basic_info' in workercinfo[container_name].keys() if basic_exist: basic_info = workercinfo[container_name]['basic_info'] + else: + basic_info['RunningTime'] = 0 + basic_info['LastTime'] = 0 + basic_info['billing'] = 0 for part in parts: if not part == '': key_val = re.split(':',part) @@ -65,11 +75,8 @@ class Container_Collector(threading.Thread): #if basic_exist: # logger.info(workercinfo[container_name]['basic_info']) if(info['State'] == 'STOPPED'): - if not 'RunningTime' in basic_info.keys(): - basic_info['RunningTime'] = 0 - basic_info['LastTime'] = 0 workercinfo[container_name]['basic_info'] = basic_info - logger.info(basic_info) + #logger.info(basic_info) return False running_time = self.get_proc_etime(int(info['PID'])) if basic_exist and 'PID' in workercinfo[container_name]['basic_info'].keys(): @@ -83,7 +90,6 @@ class Container_Collector(threading.Thread): basic_info['PID'] = info['PID'] basic_info['IP'] = info['IP'] basic_info['RunningTime'] = running_time - workercinfo[container_name]['basic_info'] = basic_info cpu_parts = re.split(' +',info['CPU use']) cpu_val = cpu_parts[0].strip() @@ -121,6 +127,11 @@ class Container_Collector(threading.Thread): cpu_use['usedp'] = cpu_usedp self.cpu_last[container_name] = cpu_val; workercinfo[container_name]['cpu_use'] = cpu_use + + if container_name not in increment.keys(): + increment[container_name] = {} + increment[container_name]['lastcputime'] = 0 + increment[container_name]['memincrement'] = 0 mem_parts = re.split(' +',info['Memory use']) mem_val = mem_parts[0].strip() @@ -130,11 +141,40 @@ class Container_Collector(threading.Thread): mem_use['unit'] = mem_unit if(mem_unit == "MiB"): mem_val = float(mem_val) * 1024 + increment[container_name]['memincrement'] += float(mem_val) elif (mem_unit == "GiB"): mem_val = float(mem_val) * 1024 * 1024 + increment[container_name]['memincrement'] += float(mem_val)*1024 mem_usedp = float(mem_val) / self.mem_quota[container_name] mem_use['usedp'] = mem_usedp workercinfo[container_name]['mem_use'] = mem_use + + lasttime = 0 + if container_name in lastbillingtime.keys(): + lasttime = lastbillingtime[container_name] + else: + lasttime = 0 + lastbillingtime[container_name] = 0 + #logger.info(running_time) + if not int(running_time/self.billingtime) == lasttime: + #logger.info("billing:"+str(float(cpu_val))) + lastbillingtime[container_name] = int(running_time/self.billingtime) + cpu_increment = float(cpu_val) - float(increment[container_name]['lastcputime']) + #logger.info("billing:"+str(cpu_increment)+" "+str(increment[container_name]['lastcputime'])) + if cpu_increment == 0.0: + avemem = 0 + else: + avemem = cpu_increment*float(increment[container_name]['memincrement'])/1800.0 + increment[container_name]['lastcputime'] = cpu_val + increment[container_name]['memincrement'] = 0 + if 'disk_use' in workercinfo[container_name].keys(): + disk_quota = workercinfo[container_name]['disk_use']['total'] + else: + disk_quota = 0 + #logger.info("cpu_increment:"+str(cpu_increment)+" avemem:"+str(avemem)+" disk:"+str(disk_quota)+"\n") + billing = cpu_increment/1000.0 + avemem/500000.0 + float(disk_quota)/1024.0/1024.0/2000 + basic_info['billing'] += math.ceil(billing) + workercinfo[container_name]['basic_info'] = basic_info #print(output) #print(parts) return True diff --git a/src/notificationmgr.py b/src/notificationmgr.py index 54619e1..6713dea 100644 --- a/src/notificationmgr.py +++ b/src/notificationmgr.py @@ -1,9 +1,14 @@ import json from log import logger -from model import db, Notification, NotificationGroups +from model import db, Notification, NotificationGroups, User from userManager import administration_required, token_required - +import smtplib +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.header import Header +from datetime import datetime +import env class NotificationMgr: def __init__(self): @@ -26,6 +31,54 @@ class NotificationMgr: notify_ids = sorted(list(set(notify_ids)), reverse=True) return [Notification.query.filter_by(id=notify_id).first() for notify_id in notify_ids] + def mail_notification(self, notify_id): + email_from_address = env.getenv('EMAIL_FROM_ADDRESS') + if (email_from_address in ['\'\'', '\"\"', '']): + return {'success' : 'true'} + notify = Notification.query.filter_by(id=notify_id).first() + notify_groups = NotificationGroups.query.filter_by(notification_id=notify_id).all() + to_addr = [] + groups = [] + for group in notify_groups: + groups.append(group.group_name) + if 'all' in groups: + users = User.query.all() + for user in users: + to_addr.append(user.e_mail) + else: + for group in notify_groups: + users = User.query.filter_by(user_group=group.group_name).all() + for user in users: + to_addr.append(user.e_mail) + + content = notify.content + text = '

Dear '+ 'user' + ':

' #user.username + ':' + text += '''

      Your account in %s has been recieved a notification:

+

%s

+
+

      Note: DO NOT reply to this email!

+

+

Docklet Team, SEI, PKU

+ ''' % (env.getenv("PORTAL_URL"), env.getenv("PORTAL_URL"), content) + text += '

'+ str(datetime.utcnow()) + '

' + text += '' + subject = 'Docklet Notification: ' + notify.title + msg = MIMEMultipart() + textmsg = MIMEText(text,'html','utf-8') + msg['Subject'] = Header(subject, 'utf-8') + msg['From'] = email_from_address + msg.attach(textmsg) + s = smtplib.SMTP() + s.connect() + for address in to_addr: + try: + msg['To'] = address + s.sendmail(email_from_address, address, msg.as_string()) + except Exception as e: + logger.error(e) + s.close() + return {"success": 'true'} + @administration_required def create_notification(self, *args, **kwargs): ''' @@ -47,6 +100,8 @@ class NotificationMgr: notify_groups = NotificationGroups(notify.id, group_name) db.session.add(notify_groups) db.session.commit() + if 'sendMail' in form: + self.mail_notification(notify.id) return {"success": 'true'} @administration_required @@ -88,6 +143,8 @@ class NotificationMgr: notify_groups = NotificationGroups(notify.id, group_name) db.session.add(notify_groups) db.session.commit() + if 'sendMail' in form: + self.mail_notification(notify_id) return {"success": 'true'} @administration_required diff --git a/web/static/js/plot_monitor.js b/web/static/js/plot_monitor.js index 0dd128c..82f45e7 100755 --- a/web/static/js/plot_monitor.js +++ b/web/static/js/plot_monitor.js @@ -206,6 +206,7 @@ function processBasicInfo() $("#con_ip").html(basic_info.IP); } $("#con_time").html(basic_info.RunningTime+"s"); + $("#con_billing").html(basic_info.billing+" beans"); },"json"); } setInterval(processBasicInfo,1000); diff --git a/web/templates/create_notification.html b/web/templates/create_notification.html index a503ff1..61fe071 100644 --- a/web/templates/create_notification.html +++ b/web/templates/create_notification.html @@ -59,6 +59,12 @@ {% endfor %} +
+ +
+ +
+
@@ -73,4 +79,4 @@ {% block script_src %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/web/templates/monitor/status.html b/web/templates/monitor/status.html index b434945..224d49d 100644 --- a/web/templates/monitor/status.html +++ b/web/templates/monitor/status.html @@ -86,6 +86,7 @@ Cpu Usage Mem Usage Disk Usage + Billing Summary @@ -105,6 +106,7 @@ -- -- -- + -- Realtime {% endfor %} @@ -135,6 +137,7 @@ $("#"+index+"_disk").html(usedp+"%
"+detail); },"json"); $("#"+index+"_time").html(data.monitor.basic_info.RunningTime+"s") + $("#"+index+"_billing").html(data.monitor.basic_info.billing+" beans") var state = data.monitor.basic_info.State; if(state == 'RUNNING') diff --git a/web/templates/monitor/statusRealtime.html b/web/templates/monitor/statusRealtime.html index a847753..a445afe 100644 --- a/web/templates/monitor/statusRealtime.html +++ b/web/templates/monitor/statusRealtime.html @@ -45,6 +45,7 @@ CPU Usage Mem Usage Disk Usage + Billing @@ -60,6 +61,7 @@ -- -- -- + {{ container['billing'] }} beans diff --git a/web/templates/notification.html b/web/templates/notification.html index b9dac6d..72314d4 100644 --- a/web/templates/notification.html +++ b/web/templates/notification.html @@ -155,6 +155,12 @@ {% endif %}
+
+
+ Also send email +
+ +
@@ -209,4 +215,4 @@ $('#deleteNotificationForm_'+notifyId).submit(); } -{% endblock %} \ No newline at end of file +{% endblock %}