Merge pull request #186 from zhongyehong/master

update base image

src/container.py

@@ -205,6 +205,15 @@ IP=%s
                     sube.output.decode('utf-8')))
             return [False, "start services for container failed"]
 
+    # mount_container: mount base image and user image by aufs
+    def mount_container(self,lxc_name):
+        logger.info ("mount container:%s" % lxc_name)
+        [success, status] = self.container_status(lxc_name)
+        if not success:
+            return [False, status]
+        self.imgmgr.checkFS(lxc_name)
+        return [True, "mount success"]
+
     # recover container: if running, do nothing. if stopped, start it
     def recover_container(self, lxc_name):
         logger.info ("recover container:%s" % lxc_name)
@@ -212,6 +221,7 @@ IP=%s
         [success, status] = self.container_status(lxc_name)
         if not success:
             return [False, status]
+        self.imgmgr.checkFS(lxc_name)
         if status == 'stopped':
             logger.info("%s stopped, recover it to running" % lxc_name)
             if self.start_container(lxc_name)[0]:
@@ -252,6 +262,16 @@ IP=%s
         #    logger.info ("stop container %s success" % lxc_name)
         #    return [True, "stop container success"]
 
+    def detach_container(self, lxc_name):
+        logger.info("detach container:%s" % lxc_name)
+        [success, status] = self.container_status(lxc_name)
+        if not success:
+            return [False, status]
+        if status == 'running':
+            logger.error("container %s is running, please stop it first" % lxc_name)
+        self.imgmgr.detachFS(lxc_name)
+        return [True, "detach container success"]
+
     # check container: check LV and mountpoints, if wrong, try to repair it
     def check_container(self, lxc_name):
         logger.info ("check container:%s" % lxc_name)
@@ -337,6 +357,9 @@ IP=%s
     def create_image(self,username,imagename,containername,description="not thing",imagenum=10):
         return self.imgmgr.createImage(username,imagename,containername,description,imagenum)
     
+    def update_basefs(self,imagename):
+        return self.imgmgr.update_basefs(imagename)
+
     # check all local containers
     def check_allcontainers(self):
         [both, onlylocal, onlyglobal] = self.diff_containers()

src/httprest.py

@@ -346,6 +346,14 @@ def list_image(cur_user, user, form):
     images = G_imagemgr.list_images(user)
     return json.dumps({'success':'true', 'images': images})
 
+@app.route("/image/updatebase/", methods=['POST'])
+@login_required
+def update_base(cur_user, user, form):
+    global G_imagemgr
+    global G_vclustermgr
+    [success, status] = G_imagemgr.update_base_image(user, G_vclustermgr, form.get('image'))
+    return json.dumps({'success':'true', 'message':status})
+
 @app.route("/image/description/", methods=['POST'])
 @login_required
 def description_image(cur_user, user, form):

src/imagemgr.py

@@ -23,6 +23,7 @@ import os,sys,subprocess,time,re,datetime,threading
 from log import logger
 import env
 from lvmtool import *
+import updatebase
 
 class ImageMgr():
     #def sys_call(self,command):
@@ -133,6 +134,9 @@ class ImageMgr():
             sys_run("mount /dev/%s/%s %s" %(vgname,lxc,layer),True)
             #self.sys_call("mkdir -p %s/overlay %s/work" % (layer,layer))
             #self.sys_call("mount -t overlay overlay -olowerdir=%s/local/basefs,upperdir=%s/overlay,workdir=%s/work %s" % (self.NFS_PREFIX,layer,layer,rootfs))
+            #self.prepareImage(user,image,layer+"/overlay")
+            self.prepareImage(user,image,layer)
+            logger.info("image has been prepared")
             sys_run("mount -t aufs -o br=%s=rw:%s/local/basefs=ro+wh -o udba=reval none %s/" % (layer,self.NFS_PREFIX,rootfs),True)
             sys_run("mkdir -p %s/local/temp/%s" % (self.NFS_PREFIX,lxc))
 
@@ -140,9 +144,6 @@ class ImageMgr():
             logger.error(e)
 
         logger.info("FS has been prepared for user:%s lxc:%s" % (user,lxc))
-        #self.prepareImage(user,image,layer+"/overlay")
-        self.prepareImage(user,image,layer)
-        logger.info("image has been prepared")
         return True
 
     def deleteFS(self,lxc,vgname="docklet-group"):
@@ -167,6 +168,14 @@ class ImageMgr():
 
         return True
    
+    def detachFS(self, lxc, vgname="docklet-group"):
+        rootfs = "/var/lib/lxc/%s/rootfs" % lxc
+        Ret = sys_run("umount %s" % rootfs)
+        if Ret.returncode != 0:
+            logger.error("cannot umount rootfs:%s" % rootfs)
+            return False
+        return True
+ 
     def checkFS(self, lxc, vgname="docklet-group"):
         rootfs = "/var/lib/lxc/%s/rootfs" % lxc
         layer = self.NFS_PREFIX + "/local/volume/" + lxc
@@ -228,6 +237,67 @@ class ImageMgr():
             sys_run("rm -f %s" % public_imgpath+"."+image+".description", True)
         except Exception as e:
             logger.error(e)
+    """
+    def update_basefs(self,image):
+        imgpath = self.imgpath + "private/root/"
+        layer = self.NFS_PREFIX + "/local/volume/update_base"
+        mountpoint = self.NFS_PREFIX + "/local/basefs_mp"
+        tmpdir = self.NFS_PREFIX + "/local/basefs_tmp"
+        olddir = self.NFS_PREFIX + "/local/basefs_old"
+        try:
+            logger.info("create directory %s, %s, %s" % (layer,mountpoint,tmpdir))
+            sys_run("mkdir -p %s" % layer)
+            sys_run("mkdir -p %s" % mountpoint)
+            sys_run("mkdir -p %s" % tmpdir)
+            logger.info("load image from %s" % imgpath+image)
+            sys_run("rsync -a --delete --exclude=lost+found/ --exclude=root/nfs/ --exclude=dev/ --exclude=mnt/ --exclude=tmp/ --exclude=media/ --exclude=proc/ --exclude=sys/ %s/ %s/" % (imgpath+image,self.dealpath(layer)),True)
+            logger.info("mount old base image and new image by aufs")
+            sys_run("mount -t aufs -o br=%s=rw:%s/local/basefs=ro+wh -o udba=reval none %s/" % (layer,self.NFS_PREFIX,mountpoint),True)
+            logger.info("save new image to %s" % tmpdir)
+            sys_run("rsync -a --delete %s/ %s/" % (self.dealpath(mountpoint),self.dealpath(tmpdir)),True)
+            logger.info("umount %s" % mountpoint)
+            sys_run("umount %s" % mountpoint)
+            logger.info("remove directory %s, %s" % (layer,mountpoint))
+            sys_run("rm -rf %s/" % mountpoint)
+            sys_run("rm -rf %s/" % layer)
+            logger.info("move old base image to an tmp directory")
+            sys_run("mv %s %s" % (self.NFS_PREFIX + "/local/basefs",olddir))
+            logger.info("move new base image from %s to %s" % (tmpdir, self.NFS_PREFIX+"/local/basefs"))
+            sys_run("mv %s %s" % (tmpdir, self.NFS_PREFIX+"/local/basefs"))
+            logger.info("remove old base image")
+            sys_run("rm -rf %s/" % olddir)
+            logger.info("update base image success")
+        except Exception as e:
+            logger.error(e)
+        return True
+    """ 
+
+    def update_basefs(self,image):
+        imgpath = self.imgpath + "private/root/"
+        basefs = self.NFS_PREFIX+"/local/basefs/"
+        try:
+            logger.info("start updating base image")
+            updatebase.aufs_update_base(imgpath+image, basefs)
+            logger.info("update base image success")
+        except Exception as e:
+            logger.error(e)
+        return True
+    
+    def update_base_image(self, user, vclustermgr, image):
+        if not user == "root":
+            logger.info("only root can update base image")
+        #vclustermgr.stop_allclusters()
+        #vclustermgr.detach_allclusters()
+        workers = vclustermgr.nodemgr.get_rpcs()
+        logger.info("update base image in all workers")
+        for worker in workers:
+            worker.update_basefs(image)
+        logger.info("update base image success")
+        #vclustermgr.mount_allclusters()
+        #logger.info("mount all cluster success")
+        #vclustermgr.recover_allclusters()
+        #logger.info("recover all cluster success")
+        return [True, "update base image"]
 
     def get_image_info(self, user, image, imagetype):
         if imagetype == "private": 

src/updatebase.py

@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+
+import os, shutil
+from log import logger
+
+def aufs_remove(basefs):
+    try:
+        if os.path.isdir(basefs):
+            shutil.rmtree(basefs)
+        elif os.path.isfile(basefs):
+            os.remove(basefs)
+    except Exception as e:
+        logger.error(e)
+
+def aufs_clean(basefs):
+    # clean the aufs mark 
+    allfiles = os.listdir(basefs)
+    for onefile in allfiles:
+        if onefile[:4] == ".wh.":
+            aufs_remove(basefs + "/" + onefile)
+
+def aufs_merge(image, basefs):
+    allfiles = os.listdir(image)
+    if ".wh..wh..opq" in allfiles:
+        #this is a new dir in image, remove the dir in basefs with the same name, and copy it to basefs
+        shutil.rmtree(basefs)
+        shutil.copytree(image, basefs, symlinks=True)
+        aufs_clean(basefs)
+        return
+    for onefile in allfiles:
+        try:
+            if onefile[:7] == ".wh..wh":
+                # aufs mark, but not white-out mark, ignore it
+                continue
+            elif onefile[:4] == ".wh.":
+                # white-out mark, remove the file in basefs
+                aufs_remove(basefs + "/" + onefile[4:])
+            elif os.path.isdir(image + "/" + onefile):
+                if os.path.isdir(basefs + "/" + onefile):
+                    # this is a dir in image and basefs, merge it
+                    aufs_merge(image + "/" + onefile, basefs + "/" + onefile)
+                elif os.path.isfile(basefs + "/" + onefile):
+                    # this is a dir in image but file in basefs, remove the file and copy the dir to basefs
+                    os.remove(basefs + "/" + onefile)
+                    shutil.copytree(image + "/" + onefile, basefs + "/" + onefile, symlinks=True)
+                elif not os.path.exists(basefs + "/" + onefile):
+                    # this is a dir in image but not exists in basefs, copy the dir to basefs
+                    shutil.copytree(image + "/" + onefile, basefs + "/" + onefile, symlinks=True)
+                else:
+                    # error
+                    logger.error(basefs + "/" + onefile + " cause error")
+            elif os.path.isfile(image + "/" + onefile):
+                if os.path.isdir(basefs + "/" + onefile):
+                    # this is a file in image but dir in basefs, remove the dir and copy the file to basefs
+                    shutil.rmtree(basefs + "/" + onefile)
+                    shutil.copy2(image+ "/" + onefile, basefs + "/" + onefile, follow_symlinks=False)
+                elif os.path.isfile(basefs + "/" + onefile):
+                    # this is a file in image and basefs, remove the file and copy the file to basefs
+                    os.remove(basefs + "/" + onefile)
+                    shutil.copy2(image+ "/" + onefile, basefs + "/" + onefile, follow_symlinks=False)
+                elif not os.path.isdir(basefs + "/" + onefile):
+                    # this is a file in image but not exists in basefs, copy the file to basefs
+                    shutil.copy2(image+ "/" + onefile, basefs + "/" + onefile, follow_symlinks=False)
+                else:
+                    # error
+                    logger.error(basefs + "/" + onefile + " cause error")
+        except Exception as e:
+            logger.error(e)
+
+def aufs_update_base(image, basefs):
+    if not os.path.isdir(basefs):
+        logger.error("basefs:%s doesn't exists" % basefs)
+    if not os.path.isdir(image):
+        logger.error("image:%s doesn't exists" % image)
+    aufs_merge(image, basefs)

src/vclustermgr.py

@@ -52,6 +52,33 @@ class VclusterMgr(object):
                 self.recover_cluster(cluster, user)
         logger.info("recovered all vclusters for all users")
     
+    def mount_allclusters(self):
+        logger.info("mounting all vclusters for all users...")
+        usersdir = self.fspath+"/global/users/"
+        for user in os.listdir(usersdir):
+            for cluster in self.list_clusters(user)[1]:
+                logger.info ("mounting cluster:%s for user:%s ..." % (cluster, user))
+                self.mount_cluster(cluster, user)
+        logger.info("mounted all vclusters for all users")
+
+    def stop_allclusters(self):
+        logger.info("stopping all vclusters for all users...")
+        usersdir = self.fspath+"/global/users/"
+        for user in os.listdir(usersdir):
+            for cluster in self.list_clusters(user)[1]:
+                logger.info ("stopping cluster:%s for user:%s ..." % (cluster, user))
+                self.stop_cluster(cluster, user)
+        logger.info("stopped all vclusters for all users")
+    
+    def detach_allclusters(self):
+        logger.info("detaching all vclusters for all users...")
+        usersdir = self.fspath+"/global/users/"
+        for user in os.listdir(usersdir):
+            for cluster in self.list_clusters(user)[1]:
+                logger.info ("detaching cluster:%s for user:%s ..." % (cluster, user))
+                self.detach_cluster(cluster, user)
+        logger.info("detached all vclusters for all users")
+     
     def create_cluster(self, clustername, username, image, user_info, setting):
         if self.is_cluster(clustername, username):
             return [False, "cluster:%s already exists" % clustername]
@@ -325,6 +352,15 @@ class VclusterMgr(object):
         infofile.close()
         return [True, "start cluster"]
 
+    def mount_cluster(self, clustername, username):
+        [status, info] = self.get_clusterinfo(clustername, username)
+        if not status:
+            return [False, "cluster not found"]
+        for container in info['containers']:
+            worker = self.nodemgr.ip_to_rpc(container['host'])
+            worker.mount_container(container['containername'])
+        return [True, "mount cluster"]
+
     def recover_cluster(self, clustername, username):
         [status, info] = self.get_clusterinfo(clustername, username)
         if not status:
@@ -363,6 +399,17 @@ class VclusterMgr(object):
         infofile.close()
         return [True, "stop cluster"]
     
+    def detach_cluster(self, clustername, username):
+        [status, info] = self.get_clusterinfo(clustername, username)
+        if not status:
+            return [False, "cluster not found"]
+        if info['status'] == 'running':
+            return [False, 'cluster is running, please stop it first']
+        for container in info['containers']:
+            worker = self.nodemgr.ip_to_rpc(container['host'])
+            worker.detach_container(container['containername'])
+        return [True, "detach cluster"]
+
     def list_clusters(self, user):
         if not os.path.exists(self.fspath+"/global/users/"+user+"/clusters"):
             return [True, []]

src/worker.py

@@ -182,7 +182,7 @@ class Worker(object):
             if status:
                 # master has know the worker so we start send heartbeat package
                 if value=='ok':
-                    self.etcd.setkey("machines/runnodes/"+self.addr, "ok", ttl = 2)
+                    self.etcd.setkey("machines/runnodes/"+self.addr, "ok", ttl = 60)
             else:
                 logger.error("get key %s failed, master crashed or initialized. restart worker please." % self.addr)
                 sys.exit(1)

web/templates/admin.html

@@ -201,8 +201,60 @@
        </div>
      </div>
  </div>
+    <div class="col-md-12">
+        <div class="box box-info">
+            <div class="box-header with-border">
+                <h3 class="box-title">Update Base Image</h3>
+                    <div class="box-tools pull-right">
+                        <button type="button" class="btn btn-box-tool" data-widget="collapse"><i class="fa fa-minus"></i></button>
+                        <button type="button" class="btn btn-box-tool" data-widget="remove"><i class="fa fa-times"></i></button>
+                    </div>
+            </div>
+            <div class="box-body">
+                <div class="table table-responsive">
+                    <table id="imageTable" class="table table-striped table-bordered">
+                        <thead>
+                            <tr>
+                                <th>ImageName</th>
+                                <th>CreateTime</th>
+                                <th>Description</th>
+                                <th>Operation</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for image in root_image %}
+                            <tr>
+                                <td>{{image['name']}}</td>
+                                <td>{{image['time']}}</td>
+                                <td><a href="/image/description/{{image['name']}}_root_private/" target="_blank">{{image['description']}}</a></td>
+                                <td><button type="button" class="btn btn-xs btn-success" data-toggle="modal" data-target="#Update_{{image['name']}}">Update</button>
+                                <div class="modal inmodal" id="Update_{{image['name']}}" tabindex="-1" role="dialog" aria-hidden="true">
+                                    <div class="modal-dialog">
+                                        <div class="modal-content animated fadeIn">
+                                            <div class="modal-header">
+                                                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                                                <i class="fa fa-save modal-icon"></i>
+                                                <h4 class="modal-title">Update Base Image</h4>
+                                                <small class="font-bold">Update Base Image From Chosen Image</small>
+                                            </div>
+                                            <div class="modal-body">
+                                                <strong>Warning: This operation will update the base image. Maybe it will cause some error and then the base image will be destroyed. Please make sure you have the backup of base image.</strong>
+                                            </div>
+                                            <div class="modal-footer">
+                                                <button type="button" class="btn btn-white" data-dismiss="modal">Close</button>
+                                                <a href="/image/updatebase/{{image['name']}}/"><button type="button" class="btn btn-success">Update</button></a>
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
+                            </tr>
+                            {% endfor %}
+                        </tbody>
+                     </table>
+                 </div>
+             </div>
+         </div>
      </div>
-<div class="row">
  <div class="col-md-12">
      <div class="box box-info">
          <div class="box-header with-border">

web/web.py

@@ -235,6 +235,12 @@ def deleteImage(image):
     deleteImageView.image = image
     return deleteImageView.as_view()
 
+@app.route("/image/updatebase/<image>/", methods=['GET'])
+@login_required
+def updatebaseImage(image):
+    updatebaseImageView.image = image
+    return updatebaseImageView.as_view()
+
 @app.route("/hosts/", methods=['GET'])
 @administration_required
 def hosts():

web/webViews/admin.py

@@ -14,8 +14,9 @@ class adminView(normalView):
         quotas = result["quotas"]
         defaultgroup = result["default"]
         parms = dockletRequest.post('/system/parmList/')
+        rootimage = dockletRequest.post('/image/list/').get('images')
         lxcsetting = dockletRequest.post('/user/lxcsettingList/')['data']
-        return self.render(self.template_path, groups = groups, quotas = quotas, defaultgroup = defaultgroup, parms = parms, lxcsetting = lxcsetting)
+        return self.render(self.template_path, groups = groups, quotas = quotas, defaultgroup = defaultgroup, parms = parms, lxcsetting = lxcsetting, root_image = rootimage['private'])
 
 class groupaddView(normalView):
     @classmethod
@@ -95,4 +96,11 @@ class historydelView(normalView):
         dockletRequest.post('/system/historydel/', request.form)
         return redirect('/admin/')
 
-
+class updatebaseImageView(normalView):
+    @classmethod
+    def get(self):
+        data = {
+                "image": self.image
+        }
+        dockletRequest.post('/image/updatebase/', data)
+        return redirect("/admin/")