diff --git a/conf/docklet.conf.template b/conf/docklet.conf.template index d0aa9a8..69f20b6 100644 --- a/conf/docklet.conf.template +++ b/conf/docklet.conf.template @@ -183,6 +183,16 @@ # Only when you deploy docklet on the cloud can you set it to True # ALLOW_SCALE_OUT=False +# WARNING_DAYS: user will receive a warning email for releasing +# when his/her vcluster has been stopped for more then the days. +# Default: 7 +# WARNING_DAYS=7 + +# RELEASE_DAYS: the vcluster will be released when it has been +# stopped for more then the days. Needs to be larger then WARNING_DAYS. +# Default: 14 +# RELEASE_DAYS=14 + # ================================================== # # Batch Config diff --git a/src/master/httprest.py b/src/master/httprest.py index 4da0739..51f47df 100755 --- a/src/master/httprest.py +++ b/src/master/httprest.py @@ -29,7 +29,7 @@ from socketserver import ThreadingMixIn from utils import etcdlib, imagemgr from master import nodemgr, vclustermgr, notificationmgr, lockmgr, cloudmgr, jobmgr, taskmgr from utils.logs import logs -from master import userManager, beansapplicationmgr, monitor, sysmgr, network +from master import userManager, beansapplicationmgr, monitor, sysmgr, network, releasemgr from worker.monitor import History_Manager import threading import requests @@ -1162,6 +1162,10 @@ if __name__ == '__main__': G_imagemgr = imagemgr.ImageMgr() logger.info("imagemgr started") + G_releasemgr = releasemgr.ReleaseMgr(G_vclustermgr,G_ulockmgr,10) + G_releasemgr.start() + logger.info("releasemgr started") + logger.info("startting to listen on: ") masterip = env.getenv('MASTER_IP') logger.info("using MASTER_IP %s", masterip) diff --git a/src/master/releasemgr.py b/src/master/releasemgr.py index 4f5f104..1d75eb6 100644 --- a/src/master/releasemgr.py +++ b/src/master/releasemgr.py @@ -2,7 +2,11 @@ import threading, time from utils import env from utils.log import logger from utils.model import db, VCluster, Container -import datetime +import smtplib, datetime +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.header import Header +from master.settings import settings userpoint = "http://" + env.getenv('USER_IP') + ":" + str(env.getenv('USER_PORT')) def post_to_user(url = '/', data={}): @@ -18,8 +22,10 @@ class ReleaseMgr(threading.Thread): self.vclustermgr = vclustermgr self.ulockmgr = ulockmgr self.check_interval = check_interval - self.warning_days = 7 - self.release_days = 14 + self.warning_days = int(env.getenv("WARNING_DAYS")) + self.release_days = int(env.getenv("RELEASE_DAYS")) + if self.release_days <= self.warning_days: + self.release_days = self.warning_days+1 def _send_email(to_address, username, vcluster, days, is_released=True): email_from_address = settings.get('EMAIL_FROM_ADDRESS') @@ -35,10 +41,11 @@ class ReleaseMgr(threading.Thread):

      You can create new workspace/vcluster if you need.

''' else: - day_d = self.release_days - self.warning_days - release_date = datetime.datetime.now() + datetime.timedelta(days=day_d) + #day_d = self.release_days - (datetime.datetime.now() - vcluster.stop_time).days + release_date = vcluster.stop_time + datetime.timedelta(days=self.release_days) + day_d = (release_date - vcluster.stop_time).days rd_str = release_date.strftime("%Y-%m-%d %H:%M:%S") - text += '''

      It will be released at %s(in about %d days).

+ text += '''

      It will be released after %s(in about %d days).

      And the data in it couldn't be recoverd after releasing.

      Please start or save it before %s(in about %d days) if you want to keep the data.

''' % (rd_str, day_d, rd_str, day_d) @@ -78,6 +85,7 @@ class ReleaseMgr(threading.Thread): if vc.stop_time is None: continue days = (datetime.datetime.now() - vc.stop_time).days + if days >= self.release_days: logger.info("[ReleaseMgr] VCluster(id:%d,user:%s) has been stopped(%s) for more than %d days, it will be released." % (vc.clusterid, vc.ownername, vc.stop_time.strftime("%Y-%m-%d %H:%M:%S"), self.release_days)) @@ -85,13 +93,24 @@ class ReleaseMgr(threading.Thread): logger.info("[ReleaseMgr] %s"%str(rc_info)) groupname = rc_info['groupname'] user_info = {"data":{"id":rc_info['uid'],"groupinfo":quotas[groupname]}} - success, msg = self.vclustermgr.delete_cluster(vc.clustername, vc.ownername, user_info) - if not success: - logger.error("[ReleaseMgr] Can't release VCluster(id:%d,user:%s) for %s"%(vc.clusterid, vc.ownername, msg)) - else: - self._send_email() + self.ulockmgr.acquire(vc.ownername) + try: + success, msg = self.vclustermgr.delete_cluster(vc.clustername, vc.ownername, user_info) + if not success: + logger.error("[ReleaseMgr] Can't release VCluster(id:%d,user:%s) for %s"%(vc.clusterid, vc.ownername, msg)) + else: + self._send_email(rc_info['email'], vc.ownername, vc, days) + except Exception as err: + logger.error(err) + finally: + self.ulockmgr.release(vc.ownername) + elif days >= self.warning_days: - pass #send warning + logger.info("[ReleaseMgr] VCluster(id:%d,user:%s) has been stopped(%s) for more than %d days. A warning email will be sent to the user." + % (vc.clusterid, vc.ownername, vc.stop_time.strftime("%Y-%m-%d %H:%M:%S"), self.warning_days)) + rc_info = post_to_user("/master/user/recoverinfo/", {'username':username,'auth_key':auth_key}) + logger.info("[ReleaseMgr] %s"%str(rc_info)) + self._send_email(rc_info['email'], vc.ownername, vc, days, False) time.sleep(self.check_interval) def stop(self): diff --git a/src/master/vclustermgr.py b/src/master/vclustermgr.py index 32a825e..e264871 100755 --- a/src/master/vclustermgr.py +++ b/src/master/vclustermgr.py @@ -707,6 +707,7 @@ class VclusterMgr(object): [status, vcluster] = self.get_vcluster(clustername, username) vcluster.status = 'stopped' vcluster.start_time ="------" + vcluster.stop_time = datetime.datetime.now() if not db_commit(): return [False, "Commit Errror"] return [True, "stop cluster"] diff --git a/src/utils/env.py b/src/utils/env.py index edd892f..16eacb5 100755 --- a/src/utils/env.py +++ b/src/utils/env.py @@ -79,6 +79,10 @@ def getenv(key): return os.environ.get("ALLOCATED_PORTS","10000-65535") elif key =="ALLOW_SCALE_OUT": return os.environ.get("ALLOW_SCALE_OUT", "False") + elif key == "WARNING_DAYS": + return os.environ.get("WARNING_DAYS"."7") + elif key == "RELEASE_DAYS": + return os.environ.get("RELEASE_DAYS"."14") elif key == "BATCH_ON": return os.environ.get("BATCH_ON","True") elif key == "BATCH_MASTER_PORT": diff --git a/user/user.py b/user/user.py index fea7f41..abfbdbc 100755 --- a/user/user.py +++ b/user/user.py @@ -314,7 +314,7 @@ def get_master_recoverinfo(): return json.dumps({'success':'false', 'message':'username field is required.'}) else: user = User.query.filter_by(username=username).first() - return json.dumps({'success':'true', 'uid':user.id, 'groupname':user.user_group}) + return json.dumps({'success':'true', 'uid':user.id, 'email':user.e_mail, 'groupname':user.user_group}) @app.route("/master/user/groupinfo/", methods=['POST']) @auth_key_required