Merge pull request #318 from iteratorlee/dev_batch

Add batch job submission

src/master/jobmgr.py

@@ -1,5 +1,11 @@
+import time, threading
+import master.monitor
+
+from utils.log import initlogging, logger
+initlogging("docklet-jobmgr")
+
 class JobMgr(object):
-    
+
     # user: username
     # job: a json string
     # user submit a new job, add this job to queue and database
@@ -18,12 +24,12 @@ class JobMgr(object):
     # call get_task to get the task information
     def get_job(self, user, jobid):
         pass
-    
+
     # job: a json string
     # this is a thread to process a job
     def job_processor(self, job):
         # according the DAG of job, add task to taskmanager
-        # wait for all task completed and exit 
+        # wait for all task completed and exit
         pass
 
     # this is a thread to schedule the jobs

web/templates/base_AdminLTE.html

@@ -156,6 +156,9 @@
         <li id="nav_History">
             <a href='/history/'><i class="fa fa-history"></i> <span class="nav-label">History</span></a>
         </li>
+        <li id="nav_Batch">
+            <a href='/batch_jobs/'><i class="fa fa-tasks"></i> <span class="nav-label">Batch</span></a>
+        </li>
 
 
   {% if mysession['usergroup'] == 'root' or mysession['usergroup'] == 'admin'%}

web/templates/batch/batch_create.html

@@ -0,0 +1,241 @@
+{% extends 'base_AdminLTE.html' %}
+
+{% block title %}Docklet | Create Batch Job{% endblock %}
+
+{% block css_src %}
+<!--<style>
+.divcontent { overflow-y:scroll; height:200px;}
+</style>-->
+<link href="//cdn.bootcss.com/datatables/1.10.11/css/dataTables.bootstrap.min.css" rel="stylesheet">
+<link href="//cdn.bootcss.com/datatables/1.10.11/css/jquery.dataTables_themeroller.css" rel="stylesheet">
+<link href="/static/dist/css/modalconfig.css" rel="stylesheet">
+
+{% endblock %}
+
+{% block panel_title %}Batch Job Info{% endblock %}
+
+{% block panel_list %}
+<ol class="breadcrumb">
+  <li>
+      <a href="/dashboard/"><i class="fa fa-dashboard"></i>Home</a>
+  </li>
+</ol>
+{% endblock %}
+
+<div>
+{% block content %}
+<div class="row">
+                <div class="col-lg-12">
+                  <div class="box box-info">
+                       <div class="box-header with-border">
+                         <h3 class="box-title">Batch Job Create
+                           <button type="button" id="add_task" class="btn btn-box-tool" title="add a task"><i class="fa fa-plus"></i>
+                           </button>
+                         </h3>
+
+                         <div class="box-tools pull-right">
+                           <button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
+                           </button>
+                           <button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
+                         </div>
+                       </div>
+                        <div class="box-body">
+                <form id="form" class="form-horizontal" action="/batch_job/add/" method="POST">
+
+                    <div class="form-group"><label class="col-sm-2 control-label">Job Name</label>
+                        <div class="col-sm-10"><input type="text" class="form-control" name="job_name" id="job_name"></div>
+                    </div>
+                    <div class="hr-line-dashed"></div>
+                    <br/>
+                        <div class="form-group"><label class="col-sm-2 control-label">Priority</label>
+                        <div class="col-sm-10"><select id="priority_selector" class="form-control">
+                        {% for priority in range(10) %}
+                        <option value="{{priority}}">{{priority}}</option>
+                        {% endfor %}
+                        </select></div>
+                    </div>
+                    <br/>
+
+                    <div class="hr-line-dashed"></div>
+                    <div class="panel-group" id="accordion">
+                    <!-- Tasks -->
+                    </div>
+                    <br/>
+                    <div class="hr-line-dashed"></div>
+                    <div class="row">
+                    <div class="form-group">
+                        <div class="col-sm-4 col-sm-offset-2">
+                            <button class="btn btn-primary" type="submit">Create</button>
+                        </div>
+                    </div>
+                    </div>
+                            </form>
+                        </div>
+                    </div>
+                    </div>
+
+</div>
+</div>
+{% endblock %}
+
+{% block script_src %}
+    <!-- Custom and plugin javascript -->
+    <script src="/static/js/inspinia.js"></script>
+
+    <script src="http://cdn.bootcss.com/pace/1.0.2/pace.min.js"></script>
+
+    <!-- Steps -->
+    <script src="http://cdn.bootcss.com/jquery-steps/1.1.0/jquery.steps.min.js"></script>
+
+    <!-- Jquery Validate -->
+    <script src="http://cdn.bootcss.com/jquery-validate/1.15.0/jquery.validate.min.js"></script>
+
+
+    <script src="http://cdn.bootcss.com/datatables/1.10.11/js/jquery.dataTables.js"></script>
+    <script src="http://cdn.bootcss.com/datatables/1.10.11/js/dataTables.bootstrap.js"></script>
+    <script src="http://cdn.bootcss.com/datatables-tabletools/2.1.5/js/TableTools.min.js"></script>
+
+
+    <script type="text/javascript">
+    var task_number = 0;
+    var mapping_number = 0;
+
+    function removeTask(obj) {
+        $("#task_pannel_" + obj.id).remove();
+    }
+
+    function removeMapping(obj) {
+        $("#mapping_" + obj.id).remove();
+    }
+
+    function addMapping(obj) {
+        mapping_number += 1;
+        var table = $("#storage_mapping_" + obj.id)[0];
+        var new_mapping = table.insertRow();
+        new_mapping.id = "mapping_" + task_number + "_" + mapping_number;
+        var local_dir = new_mapping.insertCell();
+        var remote_dir = new_mapping.insertCell();
+        var source = new_mapping.insertCell();
+        var remove = new_mapping.insertCell();
+        local_dir.innerHTML = '<input type="text" class="form-control" name="mapping_local_dir_' + task_number + '_' + mapping_number + '" id="mapping_local_dir_' 
+                             + task_number + '_' + mapping_number + '" />';
+        remote_dir.innerHTML = '<input type="text" class="form-control" name="mapping_remote_dir_' + task_number + '_' + mapping_number + '" id="mapping_remote_dir_' 
+                             + task_number + '_' + mapping_number + '" />';
+        source.innerHTML = '<select class="form-control" name="mapping_source_' + task_number + '_' + mapping_number + '" id="mapping_source_' 
+                          + task_number + '_' + mapping_number + '">'
+                          +'<option>Aliyun</option><option>AWS</option></select>';
+        remove.innerHTML = '<div class="box-tool pull-left"><button type="button" id="' + task_number + '_' + mapping_number +'" onclick="removeMapping(this)" class="btn btn-xs btn-danger">'
+                          +'Remove</button></div>';
+    }
+
+    $("#add_task").click(function() {
+        task_number += 1;
+        mapping_number = 0;
+        var task_html = '';
+        task_html += 
+              '<div class="panel panel-default" id="task_pannel_' + task_number + '">'
+             +'<div class="panel-heading">'
+             +'<h4 class="panel-title">'
+             +'<a data-toggle="collapse" data-panel="#accordion" href="#collapse' + task_number + '">'
+             +'Task #' + task_number
+             +'</a><div class="box-tools pull-right"><button type="button" id="' + task_number + '" onclick="removeTask(this)" class="btn btn-box-tool"><i class="fa fa-times"></i></button></div>'
+             +'</h4></div>'
+             +'<div id="collapse' + task_number + '" class="panel-collapse collapse">'
+             +'<div class="panel-body">'
+             +'<div class="form-group">'
+             +'<label class="col-sm-2 control-label">CPU</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="cpuSetting_' + task_number + '" id="cpuSetting_' + task_number + '" value = 1  />'
+             +'</div>'
+             +'<label class="col-sm-2 control-label">Memory</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="memorySetting_' + task_number + '" id="memorySetting_' + task_number + '" value = 1024  />'
+             +'</div>MB</div>'
+             +'<div class="form-group">'
+             +'<label class="col-sm-2 control-label">GPU</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="gpuSetting_' + task_number + '" id="gpuSetting_' + task_number + '" value= 0  />'
+             +'</div>'
+             +'<label class="col-sm-2 control-label">Disk</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="diskSetting_' + task_number + '"  id="diskSetting_' + task_number + '" value= 1024  />'
+             +'</div>MB</div>'
+             +'<div class="form-group">'
+             +'<label class="col-sm-2 control-label">Instance Count</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="instCount_' + task_number + '" id="instCount_' + task_number + '" value= 1  />'
+             +'</div>'
+             +'<label class="col-sm-2 control-label">Max Retry Count</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="retryCount_' + task_number + '" id="retryCount_' + task_number + '" value= 1  />'
+             +'</div></div>'
+             +'<div class="form-group">'
+             +'<label class="col-sm-2 control-label">Source Code Address</label>'
+             +'<div class="col-sm-3"><input type="text" class="form-control" name="srcAddr_' + task_number + '" id="srcAddr_' + task_number + '"  />'
+             +'</div>'
+             +'<label class="col-sm-2 control-label">Expire Time</label>'
+             +'<div class="col-sm-3"><input type="number" class="form-control" name="expTime_' + task_number + '" id="expTime_' + task_number + '" value= 60  />'
+             +'</div>Seconds</div>'
+             +'<div class="form-group">'
+             +'<label class="col-sm-2 control-label">Dependency&nbsp<i class="fa fa-question-circle" title="The tasks that this task depends on, seperate them with commas, eg: Task_1, Task_2"></i></label>'
+             +'<div class="col-sm-3"><input type="text" class="form-control" name="dependency_' + task_number + '" id="dependency_' + task_number + '"  />'
+             +'</div>'
+             +'<label class="col-sm-2 control-label">Command</label>'
+             +'<div class="col-sm-3"><input type="text" class="form-control" name="command_' + task_number + '" id="command_' + task_number + '"  />'
+             +'</div></div>'
+                  +'<div class="form-group"><label class="col-sm-2 control-label">Image Choose</label>'
+                      +'<div class="col-sm-10">'
+                          +'<table id="imagetable" class="table table-striped table-bordered table-hover table-image" >'
+                          +'<thead>'
+                          +'<tr>'
+                          +'<th>ImageName</th>'
+                          +'<th>Type</th>'
+                          +'<th>Owner</th>'
+                          +'<th>Description</th>'
+                          +'<th>Choose</th>'
+                          +'</tr>'
+                          +'</thead>'
+                          +'<tbody>'
+                          +'<tr>'
+                          +'<td>base</td>'
+                          +'<td>public</td>'
+                          +'<td>docklet</td>'
+                          +'<td>A base image for you</td>'
+                          +'<td><div class="i-checks"><label><input type="radio" name="image_' + task_number + '" value="base_base_base" checked="checked"></label></div></td>'
+                          +'</tr>'
+                          +'{% for image in images['private'] %}'
+                          +'<tr>'
+                          +'<td>{{image['name']}}</td>'
+                          +'<td>private</td>'
+                          +'<td>{{user}}</td>'
+                          +'<td><a href="/image/{{masterips[0].split("@")[1]}}/description/{{image['name']}}_{{user}}_private/" target="_blank">{{image['description']}}</a></td>'
+                          +'<td><div class="i-checks"><label><input type="radio" name="image_' + task_number + '" value="{{image['name']}}_{{user}}_private"></label></div></td>'
+                          +'</tr>'
+                          +'{% endfor %}'
+                          +'{% for p_user,p_images in images['public'].items() %}'
+                              +'{% for image in p_images %}'
+                              +'<tr>'
+                              +'<td>{{image['name']}}</td>'
+                              +'<td>public</td>'
+                              +'<td>{{p_user}}</td>'
+                              +'<td><a href="/image/{{masterips[0].split("@")[1]}}/description/{{image['name']}}_{{p_user}}_public/" target="_blank">{{image['description']}}</a></td>'
+                              +'<td><div class="i-checks"><label><input type="radio" name="image_' + task_number + '" value="{{image['name']}}_{{p_user}}_public"></label></div></td>'
+                              +'</tr>'
+                          +'{% endfor %}'
+                          +'{% endfor %}'
+                          +'</tbody>'
+                          +'</table>'
+                      +'</div>'
+                  +'</div>'
+             +'<div class="form-group">'
+             +'<span>'
+             +'<label class="col-sm-2 contril-label">Exteranl Storage Mapping</label>'
+             +'<table class="table table-bordered" id="storage_mapping_' + task_number + '" style="display:inline;">'
+             +'<thead>'
+             +'<tr><td><button type="button" id="' + task_number + '" class="btn btn-primary btn-xs" title="add an external storage mapping" onclick="addMapping(this)">'
+             +'<i class="fa fa-plus"></i></button></td></tr>'
+             +'<tr><th style="width:217px">Local Dir</th><th style="width:217px">Remote Dir</th><th style="width:217px">source</th><th style="width:217px">Operation</th></tr>'
+             +'</thead>'
+             +'<tbody>'
+             +'</tbody>'
+             +'</table>'
+             +'</span></div>'
+             +'</div></div></div>'
+        $(task_html).appendTo("#accordion");
+    });
+    </script>
+{% endblock %}

web/templates/batch/batch_list.html

@@ -0,0 +1,63 @@
+{% extends "base_AdminLTE.html"%}
+{% block title %}Docklet | Batch Job{% endblock %}
+
+{% block panel_title %}Batch Job{% endblock %}
+
+{% block panel_list %}
+<ol class="breadcrumb">
+  <li>
+      <a href="/dashboard/"><i class="fa fa-dashboard"></i>Home</a>
+  </li>
+  <li class="active">
+      <strong>Batch Job</strong>
+  </li>
+</ol>
+{% endblock %}
+{% block content %}
+<div class="row">
+         <div class="col-lg-12">
+           <div class="box box-info">
+                <div class="box-header with-border">
+                    <h3 class="box-title">Batch Job List</h3>
+
+                  <div class="box-tools pull-right">
+                    <button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
+                    </button>
+                    <button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
+                  </div>
+                </div>
+                 <div class="box-body">
+
+					 <p>
+					 <a href="/batch_job/create/"><button type="button" class="btn btn-primary btn-sm"><i class="fa fa-plus"></i> Create Batch Job</button></a>
+					 </p>
+
+                 </div>
+                 <table class="table table-bordered">
+                    <thead>
+                        <tr>
+                            <th>ID</th>
+                            <th>Name</th>
+                            <th>Status</th>
+                            <th>Tasks</th>
+                            <th>Operations</th>
+                            <th>Running Time</th>
+                        <tr>
+                    <thead>
+                 </table>
+             </div>
+         </div>
+	</div>
+
+{% endblock %}
+{% block script_src %}
+
+<script type="text/javascript">
+function sendAdd(){
+    document.getElementById("addForm").submit();
+}
+function sendDel(){
+    document.getElementById("delForm").submit();
+}
+</script>
+{% endblock %}

web/web.py

@@ -39,6 +39,7 @@ from webViews.cloud import *
 from webViews.authenticate.auth import login_required, administration_required,activated_required
 from webViews.authenticate.register import registerView
 from webViews.authenticate.login import loginView, logoutView
+from webViews.batch import *
 import webViews.dockletrequest
 from webViews import cookie_tool
 import traceback
@@ -119,6 +120,28 @@ def redirect_dochome():
 def config():
     return configView.as_view()
 
+@app.route("/batch_jobs/", methods=['GET'])
+@login_required
+def batch_job():
+    return batchJobListView().as_view()
+
+@app.route("/batch_job/create/", methods=['GET'])
+@login_required
+def create_batch_job():
+    return createBatchJobView().as_view()
+
+@app.route("/batch_job/add/", methods=['POST'])
+@login_required
+def add_batch_job():
+    #TODO get form parameters of a job description
+    job_data = {}
+    job_data["job_name"] = request.form["job_name"]
+    return addBatchJobView().as_view()
+
+@app.route("/batch_job/state/", methods=['GET'])
+@login_required
+def state_batch_job():
+    return stateBatchJobView().as_view()
 
 @app.route("/workspace/create/", methods=['GET'])
 @activated_required

web/webViews/batch.py

@@ -0,0 +1,46 @@
+from flask import session, redirect, request
+from webViews.view import normalView
+from webViews.checkname import checkname
+from webViews.dockletrequest import dockletRequest
+
+class batchJobListView(normalView):
+    template_path = "batch/batch_list.html"
+
+    @classmethod
+    def get(self):
+        if True:
+            return self.render(self.template_path)
+        else:
+            return self.error()
+
+class createBatchJobView(normalView):
+    template_path = "batch/batch_create.html"
+    
+    @classmethod
+    def get(self):
+        masterips = dockletRequest.post_to_all()
+        images = dockletRequest.post("/image/list/",{},masterips[0].split("@")[0]).get("images")
+        if True:
+            return self.render(self.template_path, masterips=masterips, images=images)
+        else:
+            return self.error()
+
+class stateBatchJobView(normalView):
+    template_path = "batch/batch_state.html"
+
+    @classmethod
+    def get(self):
+        if True:
+            return self.render(self.template_path)
+        else:
+            return self.error()
+
+class addBatchJobView(normalView):
+    template_path = "batch/batch_list.html"
+
+    @classmethod
+    def post(self):
+        if True:
+            return self.render(self.template_path)
+        else:
+            return self.error()