Merge pull request #280 from FirmlyReality/networkbillings

Networkbillings

src/httprest.py

@@ -520,6 +520,10 @@ def user_quotainfo_monitor(user, beans, form, issue):
         logger.info("handle request: monitor/user/createdvnodes/")
         res = G_historymgr.getCreatedVNodes(user)
         return json.dumps({'success':'true', 'createdvnodes':res})
+    elif issue == 'net_stats':
+        logger.info("handle request: monitor/user/net_stats/")
+        res = G_historymgr.get_user_net_stats(user)
+        return json.dumps({'success':'true', 'net_stats':res})                
     else:
         return json.dumps({'success':'false', 'message':"Unspported Method!"})
 

src/monitor.py

@@ -74,6 +74,11 @@ workerinfo = {}
 # has the second keys same as the third keys in monitor_vnodes.
 workercinfo = {}
 
+# store the network statistics of users' gateways on current Worker.
+# key is username
+# bytes_sent and bytes_recv are the second keys
+gateways_stats = {}
+
 # only use on worker
 containerpids = []
 pid2name = {}
@@ -218,6 +223,7 @@ class Container_Collector(threading.Thread):
         nowbillingval = workercinfo[vnode_name]['basic_info']['billing']
         nowbillingval += billingval
         workercinfo[vnode_name]['basic_info']['billing'] = nowbillingval
+        workercinfo[vnode_name]['basic_info']['billing_history'] = get_billing_history(vnode_name)
         workercinfo[vnode_name]['basic_info']['billing_history']['cpu'] += billing['cpu']
         workercinfo[vnode_name]['basic_info']['billing_history']['mem'] += billing['mem']
         workercinfo[vnode_name]['basic_info']['billing_history']['disk'] += billing['disk']
@@ -266,6 +272,12 @@ class Container_Collector(threading.Thread):
                 self.net_stats[key]['errout'] = int(raw_stats[key].errin)
                 self.net_stats[key]['dropin'] = int(raw_stats[key].dropout)
                 self.net_stats[key]['dropout'] = int(raw_stats[key].dropin)
+            else:
+                if key not in gateways_stats.keys():
+                    gateways_stats[key] = {}
+                gateways_stats[key]['bytes_recv'] = int(raw_stats[key].bytes_sent)
+                gateways_stats[key]['bytes_sent'] = int(raw_stats[key].bytes_recv)
+                gateways_stats[key]['bytes_total'] = gateways_stats[key]['bytes_recv'] + gateways_stats[key]['bytes_sent']
         #logger.info(self.net_stats)
 
     # the main function to collect monitoring data of a container
@@ -565,10 +577,11 @@ class Collector(threading.Thread):
 def workerFetchInfo(master_ip):
     global workerinfo
     global workercinfo
+    global gateways_stats
     global G_masterip
     # tell the worker the ip address of the master
     G_masterip = master_ip
-    return str([workerinfo, workercinfo])
+    return str([workerinfo, workercinfo, gateways_stats])
 
 # get owner name of a container
 def get_owner(container_name):
@@ -649,8 +662,27 @@ class Master_Collector(threading.Thread):
         self.thread_stop = False
         self.nodemgr = nodemgr
         self.master_ip = master_ip
+        self.net_lastbillings = {}
+        self.bytes_per_beans = 1000000000
         return
 
+    def net_billings(self, username, now_bytes_total):
+        global monitor_vnodes
+        if not username in self.net_lastbillings.keys():
+            self.net_lastbillings[username] = 0
+        elif int(now_bytes_total/self.bytes_per_beans) < self.net_lastbillings[username]:
+            self.net_lastbillings[username] = 0
+        diff = int(now_bytes_total/self.bytes_per_beans) - self.net_lastbillings[username]
+        if diff > 0:
+            auth_key = env.getenv('AUTH_KEY')
+            data = {"owner_name":username,"billing":diff, "auth_key":auth_key}
+            header = {'Content-Type':'application/x-www-form-urlencoded'}
+            http = Http()
+            [resp,content] = http.request("http://"+self.master_ip+"/billing/beans/","POST",urlencode(data),headers = header)
+            logger.info("response from master:"+content.decode('utf-8'))
+        self.net_lastbillings[username] += diff
+        monitor_vnodes[username]['net_stats']['net_billings'] = self.net_lastbillings[username]
+
     def run(self):
         global monitor_hosts
         global monitor_vnodes
@@ -672,6 +704,12 @@ class Master_Collector(threading.Thread):
                         if not owner in monitor_vnodes.keys():
                             monitor_vnodes[owner] = {}
                         monitor_vnodes[owner][container] = info[1][container]
+                    for user in info[2].keys():
+                        if not user in monitor_vnodes.keys():
+                            continue
+                        else:
+                            monitor_vnodes[user]['net_stats'] = info[2][user]
+                            self.net_billings(user, info[2][user]['bytes_total'])
                 except Exception as err:
                     logger.warning(traceback.format_exc())
                     logger.warning(err)
@@ -912,3 +950,14 @@ class History_Manager:
             tmp = {"name":vnode.name,"billing":vnode.billing}
             res.append(tmp)
         return res
+
+    # get users' net_stats
+    def get_user_net_stats(self,owner):
+        global monitor_vnodes
+        try:
+            res = monitor_vnodes[owner]['net_stats']
+        except Exception as err:
+            logger.warning(traceback.format_exc())
+            logger.warning(err)
+            res = {}
+        return res

web/static/js/plot_monitor.js

@@ -1,6 +1,10 @@
 var mem_usedp = 0;
 var cpu_usedp = 0;
 var is_running = true;
+var ingress_rate = 0;
+var egress_rate = 0;
+var ingress_rate_limit = 0;
+var egress_rate_limit = 0;
 
 function processMemData(data)
 {
@@ -50,7 +54,20 @@ function getCpuY()
 	return cpu_usedp*100;
 }
 
-function plot_graph(container,url,processData,getY) {
+function processRate(data)
+{
+}
+function getIngressRateP()
+{
+  //alert(ingress_rate*8 / 1000.0);
+  return ingress_rate * 8 / 1000.0;
+}
+function getEgressRateP()
+{
+  return egress_rate * 8 / 1000.0;
+}
+
+function plot_graph(container,url,processData,getY,fetchdata=true, maxy=110) {
 
     //var container = $("#flot-line-chart-moving");
 
@@ -88,9 +105,10 @@ function plot_graph(container,url,processData,getY) {
         }
 
         if (data.length < maximum) {
-            $.post(url,{user:"root",key:"root"},processData,"json");
+            if(fetchdata)
-	    var y = getY();
+                $.post(url,{},processData,"json");
-            data.push(y < 0 ? 0 : y > 100 ? 100 : y);
+	          var y = getY();
+            data.push(y < 0 ? 0 : y > maxy ? maxy : y);
         }
 
         // zip the generated y values with the x values
@@ -152,7 +170,7 @@ function plot_graph(container,url,processData,getY) {
         },
         yaxis: {
             min: 0,
-            max: 110
+            max: maxy
         },
         legend: {
             show: true
@@ -227,12 +245,14 @@ function processBasicInfo()
         var secs = Math.floor(total % 3600 % 60);
         $("#con_time").html(hour+"h "+min+"m "+secs+"s");
         $("#con_billing").html("<a target='_blank' title='How to figure out it?' href='https://unias.github.io/docklet/book/en/billing/billing.html'>"+basic_info.billing+" <img src='/static/img/bean.png' /></a>");
-        $("#con_billingthishour").html("<a target='_blank' title='How to figure out it?' href='https://unias.github.io/docklet/book/en/billing/billing.html'>"+basic_info.billing_this_hour+" <img src='/static/img/bean.png' /></a>");
+        $("#con_billingthishour").html("<a target='_blank' title='How to figure out it?' href='https://unias.github.io/docklet/book/en/billing/billing.html'>"+basic_info.billing_this_hour.total+" <img src='/static/img/bean.png' /></a>");
     },"json");
     $.post(url+"/net_stats/",{},function(data){
         var net_stats = data.monitor.net_stats;
         var in_rate = parseInt(net_stats.bytes_recv_per_sec);
         var out_rate = parseInt(net_stats.bytes_sent_per_sec);
+        ingress_rate = in_rate;
+        egress_rate = out_rate;
         $("#net_in_rate").html(num2human(in_rate)+"Bps");
         $("#net_out_rate").html(num2human(out_rate)+"Bps");
         $("#net_in_bytes").html(num2human(net_stats.bytes_recv)+"B");
@@ -245,6 +265,20 @@ function processBasicInfo()
         $("#net_out_drop").html(net_stats.dropin);
     },"json");
 }
+
+function plot_net(host,monitorurl)
+{
+  var url = "http://" + host + "/user/selfQuery/";
+
+   $.post(url,{},function(data){
+      ingress_rate_limit = parseInt(data.groupinfo.input_rate_limit);
+      egress_rate_limit = parseInt(data.groupinfo.output_rate_limit);
+      plot_graph($("#ingress-chart"), monitorurl, processRate, getIngressRateP,false,ingress_rate_limit);
+      plot_graph($("#egress-chart"), monitorurl, processRate, getEgressRateP,false,egress_rate_limit*1.5);
+   },"json");
+}
+
 setInterval(processBasicInfo,1000);
 plot_graph($("#mem-chart"),url + "/mem_use/",processMemData,getMemY);
 plot_graph($("#cpu-chart"),url + "/cpu_use/",processCpuData,getCpuY);
+plot_net(host, url + "/net_stats/");

web/templates/monitor/status.html

@@ -73,6 +73,41 @@
 	   </div>
 
 {% for master in allcontainers %}
+<div class="row">
+	<div class="col-md-12">
+         <div class="box box-info">
+              <div class="box-header with-border">
+                <h3 class="box-title">Total Network Statistics @ {{master.split("@")[1]}}</h3>
+                <div class="box-tools pull-right">
+                  <button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
+                  </button>
+                  <button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
+                </div>
+              </div>
+                     <div class="box-body table-responsive">
+
+                         <table class="table table-bordered">
+                             <thead>
+                             <tr>
+                                <th>Total Bytes Sent</th>
+                                <th>Total Bytes Received</th>
+                                <th>Total Bytes Transefer</th>
+                                <th>Network Billings</th>
+                             </tr>
+                             </thead>
+                             <tbody>
+                             <tr>
+                                <td id='{{master.split("@")[1]}}_bytes_sent'>--</td>
+                                <td id='{{master.split("@")[1]}}_bytes_recv'>--</td>
+                                <td id='{{master.split("@")[1]}}_bytes_total'>--</td>
+                                <td id='{{master.split("@")[1]}}_net_billings'>--</td>
+			                       </tr>
+                             </tbody>
+                         </table>
+                     </div>
+                 </div>
+             </div>
+	   </div>
 {% for clustername, clusterinfo in allcontainers[master].items() %}
 	  <div class="row">
 	     <div class="col-md-12">
@@ -243,6 +278,31 @@
 <script src="http://cdn.bootcss.com/datatables-tabletools/2.1.5/js/TableTools.min.js"></script>
 
 <script type='text/javascript'>
+function num2human(data)
+{
+   units=['','K','M','G','T'];
+   tempdata = data/1.0;
+   //return tempdata;
+   for(var i = 1; i < units.length; ++i)
+   {
+      if( tempdata / 1000.0 > 1)
+          tempdata = tempdata/1000.0;
+      else
+          return tempdata.toFixed(2) + units[i-1];
+   }
+   return tempdata.toFixed(2) + units[4];
+}
+   function update_net_stats(url,index)
+   {
+     $.post(url,{},function(data){
+          var bytes_sent = parseInt(data.net_stats.bytes_sent);
+          var bytes_recv = parseInt(data.net_stats.bytes_recv);
+        $("#"+index+"_bytes_sent").html(num2human(bytes_sent)+"B");
+        $("#"+index+"_bytes_recv").html(num2human(bytes_recv)+"B");
+        $("#"+index+"_bytes_total").html(num2human(bytes_sent+bytes_recv)+"B");
+        $("#"+index+"_net_billings").html(data.net_stats.net_billings);
+     },"json");
+   }
    function update(url,index)
    {
 
@@ -275,10 +335,20 @@
               $("#"+index+"_billing_port").html(data.monitor.basic_info.billing_this_hour.port)
               $("#"+index+"_billing_port_use").html(data.monitor.basic_info.billing_this_hour.port_use)
             }
-            $("#"+index+"_billing_history_cpu").html(data.monitor.basic_info.billing_history.cpu)
+            if(!!data.monitor.basic_info.billing_history)
-            $("#"+index+"_billing_history_mem").html(data.monitor.basic_info.billing_history.mem)
+            {
-            $("#"+index+"_billing_history_disk").html(data.monitor.basic_info.billing_history.disk)
+              $("#"+index+"_billing_history_cpu").html(data.monitor.basic_info.billing_history.cpu);
-            $("#"+index+"_billing_history_port").html(data.monitor.basic_info.billing_history.port)
+              $("#"+index+"_billing_history_mem").html(data.monitor.basic_info.billing_history.mem);
+              $("#"+index+"_billing_history_disk").html(data.monitor.basic_info.billing_history.disk);
+              $("#"+index+"_billing_history_port").html(data.monitor.basic_info.billing_history.port);
+            }
+            else
+            {
+              $("#"+index+"_billing_history_cpu").html(0);
+              $("#"+index+"_billing_history_mem").html(0);
+              $("#"+index+"_billing_history_disk").html(0);
+              $("#"+index+"_billing_history_port").html(0);
+            }
             $("#"+index+"_billing_history_total").html(data.monitor.basic_info.billing)
 
             $("#"+index+"_billing_cpu_a").html(data.monitor.basic_info.a_cpu)
@@ -338,6 +408,8 @@
         //var url0 = "http://" + host + "/monitor/vnodes/";
 
 	{% for master in allcontainers %}
+	url = "http://" + host + "/monitor/" + '{{master.split("@")[0]}}' + "/user/net_stats/";
+  update_net_stats(url,'{{master.split("@")[1]}}')
  	{% for clustername, clusterinfo in allcontainers[master].items() %}
 	{% for container in clusterinfo['containers'] %}
 	//url = url0 + '{{ container['containername'] }}';

web/templates/monitor/statusRealtime.html

@@ -63,8 +63,8 @@
                         <td id='con_cpu'>--</td>
                         <td id='con_mem'>--</td>
                         <td id='con_disk'>--</td>
-                        <td id='con_billing'><a target='_blank' title='How to figure out it?' href='https://unias.github.io/docklet/book/en/billing/billing.html'>{{ container['billing'] }} <img src='/static/img/bean.png' /></a></td>
+                        <td id='con_billing'>--</td>
-                        <td id='con_billingthishour'><a target='_blank' title='How to figure out it?' href='https://unias.github.io/docklet/book/en/billing/billing.html'>{{ container['billing_this_hour'] }} <img src='/static/img/bean.png' /></a></td>
+                        <td id='con_billingthishour'>--</td>
                       </tr>
                     </tbody>
                 </table>
@@ -164,6 +164,46 @@
         </div>
      </div>
   </div>
+  <div class="row">
+      <div class="col-lg-6">
+        <div class="box box-info">
+             <div class="box-header with-border">
+               <h3 class="box-title">Ingress Rate(kbps):</h3>
+
+               <div class="box-tools pull-right">
+                 <button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
+                 </button>
+                 <button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
+               </div>
+             </div>
+              <div class="box-body">
+
+                  <div class="flot-chart">
+                      <div class="flot-chart-content" id="ingress-chart"></div>
+                  </div>
+              </div>
+          </div>
+      </div>
+  		<div class="col-lg-6">
+        <div class="box box-info">
+             <div class="box-header with-border">
+               <h3 class="box-title">Egress Rate(kbps):</h3>
+
+               <div class="box-tools pull-right">
+                 <button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
+                 </button>
+                 <button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
+               </div>
+             </div>
+              <div class="box-body">
+
+                  <div class="flot-chart">
+                      <div class="flot-chart-content" id="egress-chart"></div>
+                  </div>
+              </div>
+          </div>
+       </div>
+    </div>
 {% endblock %}
 
 {% block script_src %}

web/web.py

@@ -360,6 +360,18 @@ def monitor_request(comid,infotype,masterip):
     logger.debug("monitor" + str(type(result)))
     return json.dumps(result)
 
+@app.route("/monitor/<masterip>/user/<issue>/", methods=['POST'])
+@login_required
+def monitor_user_request(issue,masterip):
+    data = {
+        "user": session['username']
+    }
+    path = "/monitor/user/" + str(issue) + "/"
+    logger.debug(path + "_____" + masterip)
+    result = dockletRequest.post(path, data, masterip)
+    logger.debug("monitor" + str(type(result)))
+    return json.dumps(result)
+
 @app.route("/beans/application/", methods=['GET'])
 @login_required
 def beansapplication():
@@ -479,6 +491,12 @@ def groupdel(groupname):
 def userinfo():
     return userinfoView.as_view()
 
+@app.route("/user/selfQuery/", methods=['GET', 'POST'])
+@login_required
+def userselfQuery():
+    result = dockletRequest.post('/user/selfQuery/')
+    return json.dumps(result['data'])
+
 @app.route("/user/query/", methods=['GET', 'POST'])
 @administration_required
 def userquery():