Merge pull request #318 from iteratorlee/dev_batch
Add batch job submission
This commit is contained in:
Yan Li
GitHub
commit
30b2b88eb8
|
|
@ -1,5 +1,11 @@
|
|||
import time, threading
|
||||
import master.monitor
|
||||
|
||||
from utils.log import initlogging, logger
|
||||
initlogging("docklet-jobmgr")
|
||||
|
||||
class JobMgr(object):
|
||||
|
||||
|
||||
# user: username
|
||||
# job: a json string
|
||||
# user submit a new job, add this job to queue and database
|
||||
|
|
@ -18,12 +24,12 @@ class JobMgr(object):
|
|||
# call get_task to get the task information
|
||||
def get_job(self, user, jobid):
|
||||
pass
|
||||
|
||||
|
||||
# job: a json string
|
||||
# this is a thread to process a job
|
||||
def job_processor(self, job):
|
||||
# according the DAG of job, add task to taskmanager
|
||||
# wait for all task completed and exit
|
||||
# wait for all task completed and exit
|
||||
pass
|
||||
|
||||
# this is a thread to schedule the jobs
|
||||
|
|
|
|||
|
|
@ -156,6 +156,9 @@
|
|||
<li id="nav_History">
|
||||
<a href='/history/'><i class="fa fa-history"></i> <span class="nav-label">History</span></a>
|
||||
</li>
|
||||
<li id="nav_Batch">
|
||||
<a href='/batch_jobs/'><i class="fa fa-tasks"></i> <span class="nav-label">Batch</span></a>
|
||||
</li>
|
||||
|
||||
|
||||
{% if mysession['usergroup'] == 'root' or mysession['usergroup'] == 'admin'%}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,241 @@
|
|||
{% extends 'base_AdminLTE.html' %}
|
||||
|
||||
{% block title %}Docklet | Create Batch Job{% endblock %}
|
||||
|
||||
{% block css_src %}
|
||||
<!--<style>
|
||||
.divcontent { overflow-y:scroll; height:200px;}
|
||||
</style>-->
|
||||
<link href="//cdn.bootcss.com/datatables/1.10.11/css/dataTables.bootstrap.min.css" rel="stylesheet">
|
||||
<link href="//cdn.bootcss.com/datatables/1.10.11/css/jquery.dataTables_themeroller.css" rel="stylesheet">
|
||||
<link href="/static/dist/css/modalconfig.css" rel="stylesheet">
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block panel_title %}Batch Job Info{% endblock %}
|
||||
|
||||
{% block panel_list %}
|
||||
<ol class="breadcrumb">
|
||||
<li>
|
||||
<a href="/dashboard/"><i class="fa fa-dashboard"></i>Home</a>
|
||||
</li>
|
||||
</ol>
|
||||
{% endblock %}
|
||||
|
||||
<div>
|
||||
{% block content %}
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="box box-info">
|
||||
<div class="box-header with-border">
|
||||
<h3 class="box-title">Batch Job Create
|
||||
<button type="button" id="add_task" class="btn btn-box-tool" title="add a task"><i class="fa fa-plus"></i>
|
||||
</button>
|
||||
</h3>
|
||||
|
||||
<div class="box-tools pull-right">
|
||||
<button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box-body">
|
||||
<form id="form" class="form-horizontal" action="/batch_job/add/" method="POST">
|
||||
|
||||
<div class="form-group"><label class="col-sm-2 control-label">Job Name</label>
|
||||
<div class="col-sm-10"><input type="text" class="form-control" name="job_name" id="job_name"></div>
|
||||
</div>
|
||||
<div class="hr-line-dashed"></div>
|
||||
<br/>
|
||||
<div class="form-group"><label class="col-sm-2 control-label">Priority</label>
|
||||
<div class="col-sm-10"><select id="priority_selector" class="form-control">
|
||||
{% for priority in range(10) %}
|
||||
<option value="{{priority}}">{{priority}}</option>
|
||||
{% endfor %}
|
||||
</select></div>
|
||||
</div>
|
||||
<br/>
|
||||
|
||||
<div class="hr-line-dashed"></div>
|
||||
<div class="panel-group" id="accordion">
|
||||
<!-- Tasks -->
|
||||
</div>
|
||||
<br/>
|
||||
<div class="hr-line-dashed"></div>
|
||||
<div class="row">
|
||||
<div class="form-group">
|
||||
<div class="col-sm-4 col-sm-offset-2">
|
||||
<button class="btn btn-primary" type="submit">Create</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block script_src %}
|
||||
<!-- Custom and plugin javascript -->
|
||||
<script src="/static/js/inspinia.js"></script>
|
||||
|
||||
<script src="http://cdn.bootcss.com/pace/1.0.2/pace.min.js"></script>
|
||||
|
||||
<!-- Steps -->
|
||||
<script src="http://cdn.bootcss.com/jquery-steps/1.1.0/jquery.steps.min.js"></script>
|
||||
|
||||
<!-- Jquery Validate -->
|
||||
<script src="http://cdn.bootcss.com/jquery-validate/1.15.0/jquery.validate.min.js"></script>
|
||||
|
||||
|
||||
<script src="http://cdn.bootcss.com/datatables/1.10.11/js/jquery.dataTables.js"></script>
|
||||
<script src="http://cdn.bootcss.com/datatables/1.10.11/js/dataTables.bootstrap.js"></script>
|
||||
<script src="http://cdn.bootcss.com/datatables-tabletools/2.1.5/js/TableTools.min.js"></script>
|
||||
|
||||
|
||||
<script type="text/javascript">
|
||||
var task_number = 0;
|
||||
var mapping_number = 0;
|
||||
|
||||
function removeTask(obj) {
|
||||
$("#task_pannel_" + obj.id).remove();
|
||||
}
|
||||
|
||||
function removeMapping(obj) {
|
||||
$("#mapping_" + obj.id).remove();
|
||||
}
|
||||
|
||||
function addMapping(obj) {
|
||||
mapping_number += 1;
|
||||
var table = $("#storage_mapping_" + obj.id)[0];
|
||||
var new_mapping = table.insertRow();
|
||||
new_mapping.id = "mapping_" + task_number + "_" + mapping_number;
|
||||
var local_dir = new_mapping.insertCell();
|
||||
var remote_dir = new_mapping.insertCell();
|
||||
var source = new_mapping.insertCell();
|
||||
var remove = new_mapping.insertCell();
|
||||
local_dir.innerHTML = '<input type="text" class="form-control" name="mapping_local_dir_' + task_number + '_' + mapping_number + '" id="mapping_local_dir_'
|
||||
+ task_number + '_' + mapping_number + '" />';
|
||||
remote_dir.innerHTML = '<input type="text" class="form-control" name="mapping_remote_dir_' + task_number + '_' + mapping_number + '" id="mapping_remote_dir_'
|
||||
+ task_number + '_' + mapping_number + '" />';
|
||||
source.innerHTML = '<select class="form-control" name="mapping_source_' + task_number + '_' + mapping_number + '" id="mapping_source_'
|
||||
+ task_number + '_' + mapping_number + '">'
|
||||
+'<option>Aliyun</option><option>AWS</option></select>';
|
||||
remove.innerHTML = '<div class="box-tool pull-left"><button type="button" id="' + task_number + '_' + mapping_number +'" onclick="removeMapping(this)" class="btn btn-xs btn-danger">'
|
||||
+'Remove</button></div>';
|
||||
}
|
||||
|
||||
$("#add_task").click(function() {
|
||||
task_number += 1;
|
||||
mapping_number = 0;
|
||||
var task_html = '';
|
||||
task_html +=
|
||||
'<div class="panel panel-default" id="task_pannel_' + task_number + '">'
|
||||
+'<div class="panel-heading">'
|
||||
+'<h4 class="panel-title">'
|
||||
+'<a data-toggle="collapse" data-panel="#accordion" href="#collapse' + task_number + '">'
|
||||
+'Task #' + task_number
|
||||
+'</a><div class="box-tools pull-right"><button type="button" id="' + task_number + '" onclick="removeTask(this)" class="btn btn-box-tool"><i class="fa fa-times"></i></button></div>'
|
||||
+'</h4></div>'
|
||||
+'<div id="collapse' + task_number + '" class="panel-collapse collapse">'
|
||||
+'<div class="panel-body">'
|
||||
+'<div class="form-group">'
|
||||
+'<label class="col-sm-2 control-label">CPU</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="cpuSetting_' + task_number + '" id="cpuSetting_' + task_number + '" value = 1 />'
|
||||
+'</div>'
|
||||
+'<label class="col-sm-2 control-label">Memory</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="memorySetting_' + task_number + '" id="memorySetting_' + task_number + '" value = 1024 />'
|
||||
+'</div>MB</div>'
|
||||
+'<div class="form-group">'
|
||||
+'<label class="col-sm-2 control-label">GPU</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="gpuSetting_' + task_number + '" id="gpuSetting_' + task_number + '" value= 0 />'
|
||||
+'</div>'
|
||||
+'<label class="col-sm-2 control-label">Disk</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="diskSetting_' + task_number + '" id="diskSetting_' + task_number + '" value= 1024 />'
|
||||
+'</div>MB</div>'
|
||||
+'<div class="form-group">'
|
||||
+'<label class="col-sm-2 control-label">Instance Count</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="instCount_' + task_number + '" id="instCount_' + task_number + '" value= 1 />'
|
||||
+'</div>'
|
||||
+'<label class="col-sm-2 control-label">Max Retry Count</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="retryCount_' + task_number + '" id="retryCount_' + task_number + '" value= 1 />'
|
||||
+'</div></div>'
|
||||
+'<div class="form-group">'
|
||||
+'<label class="col-sm-2 control-label">Source Code Address</label>'
|
||||
+'<div class="col-sm-3"><input type="text" class="form-control" name="srcAddr_' + task_number + '" id="srcAddr_' + task_number + '" />'
|
||||
+'</div>'
|
||||
+'<label class="col-sm-2 control-label">Expire Time</label>'
|
||||
+'<div class="col-sm-3"><input type="number" class="form-control" name="expTime_' + task_number + '" id="expTime_' + task_number + '" value= 60 />'
|
||||
+'</div>Seconds</div>'
|
||||
+'<div class="form-group">'
|
||||
+'<label class="col-sm-2 control-label">Dependency <i class="fa fa-question-circle" title="The tasks that this task depends on, seperate them with commas, eg: Task_1, Task_2"></i></label>'
|
||||
+'<div class="col-sm-3"><input type="text" class="form-control" name="dependency_' + task_number + '" id="dependency_' + task_number + '" />'
|
||||
+'</div>'
|
||||
+'<label class="col-sm-2 control-label">Command</label>'
|
||||
+'<div class="col-sm-3"><input type="text" class="form-control" name="command_' + task_number + '" id="command_' + task_number + '" />'
|
||||
+'</div></div>'
|
||||
+'<div class="form-group"><label class="col-sm-2 control-label">Image Choose</label>'
|
||||
+'<div class="col-sm-10">'
|
||||
+'<table id="imagetable" class="table table-striped table-bordered table-hover table-image" >'
|
||||
+'<thead>'
|
||||
+'<tr>'
|
||||
+'<th>ImageName</th>'
|
||||
+'<th>Type</th>'
|
||||
+'<th>Owner</th>'
|
||||
+'<th>Description</th>'
|
||||
+'<th>Choose</th>'
|
||||
+'</tr>'
|
||||
+'</thead>'
|
||||
+'<tbody>'
|
||||
+'<tr>'
|
||||
+'<td>base</td>'
|
||||
+'<td>public</td>'
|
||||
+'<td>docklet</td>'
|
||||
+'<td>A base image for you</td>'
|
||||
+'<td><div class="i-checks"><label><input type="radio" name="image_' + task_number + '" value="base_base_base" checked="checked"></label></div></td>'
|
||||
+'</tr>'
|
||||
+'{% for image in images['private'] %}'
|
||||
+'<tr>'
|
||||
+'<td>{{image['name']}}</td>'
|
||||
+'<td>private</td>'
|
||||
+'<td>{{user}}</td>'
|
||||
+'<td><a href="/image/{{masterips[0].split("@")[1]}}/description/{{image['name']}}_{{user}}_private/" target="_blank">{{image['description']}}</a></td>'
|
||||
+'<td><div class="i-checks"><label><input type="radio" name="image_' + task_number + '" value="{{image['name']}}_{{user}}_private"></label></div></td>'
|
||||
+'</tr>'
|
||||
+'{% endfor %}'
|
||||
+'{% for p_user,p_images in images['public'].items() %}'
|
||||
+'{% for image in p_images %}'
|
||||
+'<tr>'
|
||||
+'<td>{{image['name']}}</td>'
|
||||
+'<td>public</td>'
|
||||
+'<td>{{p_user}}</td>'
|
||||
+'<td><a href="/image/{{masterips[0].split("@")[1]}}/description/{{image['name']}}_{{p_user}}_public/" target="_blank">{{image['description']}}</a></td>'
|
||||
+'<td><div class="i-checks"><label><input type="radio" name="image_' + task_number + '" value="{{image['name']}}_{{p_user}}_public"></label></div></td>'
|
||||
+'</tr>'
|
||||
+'{% endfor %}'
|
||||
+'{% endfor %}'
|
||||
+'</tbody>'
|
||||
+'</table>'
|
||||
+'</div>'
|
||||
+'</div>'
|
||||
+'<div class="form-group">'
|
||||
+'<span>'
|
||||
+'<label class="col-sm-2 contril-label">Exteranl Storage Mapping</label>'
|
||||
+'<table class="table table-bordered" id="storage_mapping_' + task_number + '" style="display:inline;">'
|
||||
+'<thead>'
|
||||
+'<tr><td><button type="button" id="' + task_number + '" class="btn btn-primary btn-xs" title="add an external storage mapping" onclick="addMapping(this)">'
|
||||
+'<i class="fa fa-plus"></i></button></td></tr>'
|
||||
+'<tr><th style="width:217px">Local Dir</th><th style="width:217px">Remote Dir</th><th style="width:217px">source</th><th style="width:217px">Operation</th></tr>'
|
||||
+'</thead>'
|
||||
+'<tbody>'
|
||||
+'</tbody>'
|
||||
+'</table>'
|
||||
+'</span></div>'
|
||||
+'</div></div></div>'
|
||||
$(task_html).appendTo("#accordion");
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
{% extends "base_AdminLTE.html"%}
|
||||
{% block title %}Docklet | Batch Job{% endblock %}
|
||||
|
||||
{% block panel_title %}Batch Job{% endblock %}
|
||||
|
||||
{% block panel_list %}
|
||||
<ol class="breadcrumb">
|
||||
<li>
|
||||
<a href="/dashboard/"><i class="fa fa-dashboard"></i>Home</a>
|
||||
</li>
|
||||
<li class="active">
|
||||
<strong>Batch Job</strong>
|
||||
</li>
|
||||
</ol>
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="box box-info">
|
||||
<div class="box-header with-border">
|
||||
<h3 class="box-title">Batch Job List</h3>
|
||||
|
||||
<div class="box-tools pull-right">
|
||||
<button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box-body">
|
||||
|
||||
<p>
|
||||
<a href="/batch_job/create/"><button type="button" class="btn btn-primary btn-sm"><i class="fa fa-plus"></i> Create Batch Job</button></a>
|
||||
</p>
|
||||
|
||||
</div>
|
||||
<table class="table table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Name</th>
|
||||
<th>Status</th>
|
||||
<th>Tasks</th>
|
||||
<th>Operations</th>
|
||||
<th>Running Time</th>
|
||||
<tr>
|
||||
<thead>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
{% block script_src %}
|
||||
|
||||
<script type="text/javascript">
|
||||
function sendAdd(){
|
||||
document.getElementById("addForm").submit();
|
||||
}
|
||||
function sendDel(){
|
||||
document.getElementById("delForm").submit();
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
23
web/web.py
23
web/web.py
|
|
@ -39,6 +39,7 @@ from webViews.cloud import *
|
|||
from webViews.authenticate.auth import login_required, administration_required,activated_required
|
||||
from webViews.authenticate.register import registerView
|
||||
from webViews.authenticate.login import loginView, logoutView
|
||||
from webViews.batch import *
|
||||
import webViews.dockletrequest
|
||||
from webViews import cookie_tool
|
||||
import traceback
|
||||
|
|
@ -119,6 +120,28 @@ def redirect_dochome():
|
|||
def config():
|
||||
return configView.as_view()
|
||||
|
||||
@app.route("/batch_jobs/", methods=['GET'])
|
||||
@login_required
|
||||
def batch_job():
|
||||
return batchJobListView().as_view()
|
||||
|
||||
@app.route("/batch_job/create/", methods=['GET'])
|
||||
@login_required
|
||||
def create_batch_job():
|
||||
return createBatchJobView().as_view()
|
||||
|
||||
@app.route("/batch_job/add/", methods=['POST'])
|
||||
@login_required
|
||||
def add_batch_job():
|
||||
#TODO get form parameters of a job description
|
||||
job_data = {}
|
||||
job_data["job_name"] = request.form["job_name"]
|
||||
return addBatchJobView().as_view()
|
||||
|
||||
@app.route("/batch_job/state/", methods=['GET'])
|
||||
@login_required
|
||||
def state_batch_job():
|
||||
return stateBatchJobView().as_view()
|
||||
|
||||
@app.route("/workspace/create/", methods=['GET'])
|
||||
@activated_required
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
from flask import session, redirect, request
|
||||
from webViews.view import normalView
|
||||
from webViews.checkname import checkname
|
||||
from webViews.dockletrequest import dockletRequest
|
||||
|
||||
class batchJobListView(normalView):
|
||||
template_path = "batch/batch_list.html"
|
||||
|
||||
@classmethod
|
||||
def get(self):
|
||||
if True:
|
||||
return self.render(self.template_path)
|
||||
else:
|
||||
return self.error()
|
||||
|
||||
class createBatchJobView(normalView):
|
||||
template_path = "batch/batch_create.html"
|
||||
|
||||
@classmethod
|
||||
def get(self):
|
||||
masterips = dockletRequest.post_to_all()
|
||||
images = dockletRequest.post("/image/list/",{},masterips[0].split("@")[0]).get("images")
|
||||
if True:
|
||||
return self.render(self.template_path, masterips=masterips, images=images)
|
||||
else:
|
||||
return self.error()
|
||||
|
||||
class stateBatchJobView(normalView):
|
||||
template_path = "batch/batch_state.html"
|
||||
|
||||
@classmethod
|
||||
def get(self):
|
||||
if True:
|
||||
return self.render(self.template_path)
|
||||
else:
|
||||
return self.error()
|
||||
|
||||
class addBatchJobView(normalView):
|
||||
template_path = "batch/batch_list.html"
|
||||
|
||||
@classmethod
|
||||
def post(self):
|
||||
if True:
|
||||
return self.render(self.template_path)
|
||||
else:
|
||||
return self.error()
|
||||
Loading…
Reference in New Issue