diff --git a/src/container.py b/src/container.py index 2895509..d4bf9ca 100755 --- a/src/container.py +++ b/src/container.py @@ -205,6 +205,15 @@ IP=%s sube.output.decode('utf-8'))) return [False, "start services for container failed"] + # mount_container: mount base image and user image by aufs + def mount_container(self,lxc_name): + logger.info ("mount container:%s" % lxc_name) + [success, status] = self.container_status(lxc_name) + if not success: + return [False, status] + self.imgmgr.checkFS(lxc_name) + return [True, "mount success"] + # recover container: if running, do nothing. if stopped, start it def recover_container(self, lxc_name): logger.info ("recover container:%s" % lxc_name) @@ -212,6 +221,7 @@ IP=%s [success, status] = self.container_status(lxc_name) if not success: return [False, status] + self.imgmgr.checkFS(lxc_name) if status == 'stopped': logger.info("%s stopped, recover it to running" % lxc_name) if self.start_container(lxc_name)[0]: @@ -252,6 +262,16 @@ IP=%s # logger.info ("stop container %s success" % lxc_name) # return [True, "stop container success"] + def detach_container(self, lxc_name): + logger.info("detach container:%s" % lxc_name) + [success, status] = self.container_status(lxc_name) + if not success: + return [False, status] + if status == 'running': + logger.error("container %s is running, please stop it first" % lxc_name) + self.imgmgr.detachFS(lxc_name) + return [True, "detach container success"] + # check container: check LV and mountpoints, if wrong, try to repair it def check_container(self, lxc_name): logger.info ("check container:%s" % lxc_name) @@ -336,6 +356,9 @@ IP=%s def create_image(self,username,imagename,containername,description="not thing",imagenum=10): return self.imgmgr.createImage(username,imagename,containername,description,imagenum) + + def update_basefs(self,imagename): + return self.imgmgr.update_basefs(imagename) # check all local containers def check_allcontainers(self): diff --git a/src/httprest.py b/src/httprest.py index 640af2f..ffc4e3f 100755 --- a/src/httprest.py +++ b/src/httprest.py @@ -346,6 +346,14 @@ def list_image(cur_user, user, form): images = G_imagemgr.list_images(user) return json.dumps({'success':'true', 'images': images}) +@app.route("/image/updatebase/", methods=['POST']) +@login_required +def update_base(cur_user, user, form): + global G_imagemgr + global G_vclustermgr + [success, status] = G_imagemgr.update_base_image(user, G_vclustermgr, form.get('image')) + return json.dumps({'success':'true', 'message':status}) + @app.route("/image/description/", methods=['POST']) @login_required def description_image(cur_user, user, form): diff --git a/src/imagemgr.py b/src/imagemgr.py index 5c45fd9..0592f6e 100755 --- a/src/imagemgr.py +++ b/src/imagemgr.py @@ -23,6 +23,7 @@ import os,sys,subprocess,time,re,datetime,threading from log import logger import env from lvmtool import * +import updatebase class ImageMgr(): #def sys_call(self,command): @@ -133,6 +134,9 @@ class ImageMgr(): sys_run("mount /dev/%s/%s %s" %(vgname,lxc,layer),True) #self.sys_call("mkdir -p %s/overlay %s/work" % (layer,layer)) #self.sys_call("mount -t overlay overlay -olowerdir=%s/local/basefs,upperdir=%s/overlay,workdir=%s/work %s" % (self.NFS_PREFIX,layer,layer,rootfs)) + #self.prepareImage(user,image,layer+"/overlay") + self.prepareImage(user,image,layer) + logger.info("image has been prepared") sys_run("mount -t aufs -o br=%s=rw:%s/local/basefs=ro+wh -o udba=reval none %s/" % (layer,self.NFS_PREFIX,rootfs),True) sys_run("mkdir -p %s/local/temp/%s" % (self.NFS_PREFIX,lxc)) @@ -140,9 +144,6 @@ class ImageMgr(): logger.error(e) logger.info("FS has been prepared for user:%s lxc:%s" % (user,lxc)) - #self.prepareImage(user,image,layer+"/overlay") - self.prepareImage(user,image,layer) - logger.info("image has been prepared") return True def deleteFS(self,lxc,vgname="docklet-group"): @@ -166,7 +167,15 @@ class ImageMgr(): logger.error(e) return True - + + def detachFS(self, lxc, vgname="docklet-group"): + rootfs = "/var/lib/lxc/%s/rootfs" % lxc + Ret = sys_run("umount %s" % rootfs) + if Ret.returncode != 0: + logger.error("cannot umount rootfs:%s" % rootfs) + return False + return True + def checkFS(self, lxc, vgname="docklet-group"): rootfs = "/var/lib/lxc/%s/rootfs" % lxc layer = self.NFS_PREFIX + "/local/volume/" + lxc @@ -228,6 +237,67 @@ class ImageMgr(): sys_run("rm -f %s" % public_imgpath+"."+image+".description", True) except Exception as e: logger.error(e) + """ + def update_basefs(self,image): + imgpath = self.imgpath + "private/root/" + layer = self.NFS_PREFIX + "/local/volume/update_base" + mountpoint = self.NFS_PREFIX + "/local/basefs_mp" + tmpdir = self.NFS_PREFIX + "/local/basefs_tmp" + olddir = self.NFS_PREFIX + "/local/basefs_old" + try: + logger.info("create directory %s, %s, %s" % (layer,mountpoint,tmpdir)) + sys_run("mkdir -p %s" % layer) + sys_run("mkdir -p %s" % mountpoint) + sys_run("mkdir -p %s" % tmpdir) + logger.info("load image from %s" % imgpath+image) + sys_run("rsync -a --delete --exclude=lost+found/ --exclude=root/nfs/ --exclude=dev/ --exclude=mnt/ --exclude=tmp/ --exclude=media/ --exclude=proc/ --exclude=sys/ %s/ %s/" % (imgpath+image,self.dealpath(layer)),True) + logger.info("mount old base image and new image by aufs") + sys_run("mount -t aufs -o br=%s=rw:%s/local/basefs=ro+wh -o udba=reval none %s/" % (layer,self.NFS_PREFIX,mountpoint),True) + logger.info("save new image to %s" % tmpdir) + sys_run("rsync -a --delete %s/ %s/" % (self.dealpath(mountpoint),self.dealpath(tmpdir)),True) + logger.info("umount %s" % mountpoint) + sys_run("umount %s" % mountpoint) + logger.info("remove directory %s, %s" % (layer,mountpoint)) + sys_run("rm -rf %s/" % mountpoint) + sys_run("rm -rf %s/" % layer) + logger.info("move old base image to an tmp directory") + sys_run("mv %s %s" % (self.NFS_PREFIX + "/local/basefs",olddir)) + logger.info("move new base image from %s to %s" % (tmpdir, self.NFS_PREFIX+"/local/basefs")) + sys_run("mv %s %s" % (tmpdir, self.NFS_PREFIX+"/local/basefs")) + logger.info("remove old base image") + sys_run("rm -rf %s/" % olddir) + logger.info("update base image success") + except Exception as e: + logger.error(e) + return True + """ + + def update_basefs(self,image): + imgpath = self.imgpath + "private/root/" + basefs = self.NFS_PREFIX+"/local/basefs/" + try: + logger.info("start updating base image") + updatebase.aufs_update_base(imgpath+image, basefs) + logger.info("update base image success") + except Exception as e: + logger.error(e) + return True + + def update_base_image(self, user, vclustermgr, image): + if not user == "root": + logger.info("only root can update base image") + #vclustermgr.stop_allclusters() + #vclustermgr.detach_allclusters() + workers = vclustermgr.nodemgr.get_rpcs() + logger.info("update base image in all workers") + for worker in workers: + worker.update_basefs(image) + logger.info("update base image success") + #vclustermgr.mount_allclusters() + #logger.info("mount all cluster success") + #vclustermgr.recover_allclusters() + #logger.info("recover all cluster success") + return [True, "update base image"] def get_image_info(self, user, image, imagetype): if imagetype == "private": diff --git a/src/updatebase.py b/src/updatebase.py new file mode 100755 index 0000000..a811c8b --- /dev/null +++ b/src/updatebase.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 + +import os, shutil +from log import logger + +def aufs_remove(basefs): + try: + if os.path.isdir(basefs): + shutil.rmtree(basefs) + elif os.path.isfile(basefs): + os.remove(basefs) + except Exception as e: + logger.error(e) + +def aufs_clean(basefs): + # clean the aufs mark + allfiles = os.listdir(basefs) + for onefile in allfiles: + if onefile[:4] == ".wh.": + aufs_remove(basefs + "/" + onefile) + +def aufs_merge(image, basefs): + allfiles = os.listdir(image) + if ".wh..wh..opq" in allfiles: + #this is a new dir in image, remove the dir in basefs with the same name, and copy it to basefs + shutil.rmtree(basefs) + shutil.copytree(image, basefs, symlinks=True) + aufs_clean(basefs) + return + for onefile in allfiles: + try: + if onefile[:7] == ".wh..wh": + # aufs mark, but not white-out mark, ignore it + continue + elif onefile[:4] == ".wh.": + # white-out mark, remove the file in basefs + aufs_remove(basefs + "/" + onefile[4:]) + elif os.path.isdir(image + "/" + onefile): + if os.path.isdir(basefs + "/" + onefile): + # this is a dir in image and basefs, merge it + aufs_merge(image + "/" + onefile, basefs + "/" + onefile) + elif os.path.isfile(basefs + "/" + onefile): + # this is a dir in image but file in basefs, remove the file and copy the dir to basefs + os.remove(basefs + "/" + onefile) + shutil.copytree(image + "/" + onefile, basefs + "/" + onefile, symlinks=True) + elif not os.path.exists(basefs + "/" + onefile): + # this is a dir in image but not exists in basefs, copy the dir to basefs + shutil.copytree(image + "/" + onefile, basefs + "/" + onefile, symlinks=True) + else: + # error + logger.error(basefs + "/" + onefile + " cause error") + elif os.path.isfile(image + "/" + onefile): + if os.path.isdir(basefs + "/" + onefile): + # this is a file in image but dir in basefs, remove the dir and copy the file to basefs + shutil.rmtree(basefs + "/" + onefile) + shutil.copy2(image+ "/" + onefile, basefs + "/" + onefile, follow_symlinks=False) + elif os.path.isfile(basefs + "/" + onefile): + # this is a file in image and basefs, remove the file and copy the file to basefs + os.remove(basefs + "/" + onefile) + shutil.copy2(image+ "/" + onefile, basefs + "/" + onefile, follow_symlinks=False) + elif not os.path.isdir(basefs + "/" + onefile): + # this is a file in image but not exists in basefs, copy the file to basefs + shutil.copy2(image+ "/" + onefile, basefs + "/" + onefile, follow_symlinks=False) + else: + # error + logger.error(basefs + "/" + onefile + " cause error") + except Exception as e: + logger.error(e) + +def aufs_update_base(image, basefs): + if not os.path.isdir(basefs): + logger.error("basefs:%s doesn't exists" % basefs) + if not os.path.isdir(image): + logger.error("image:%s doesn't exists" % image) + aufs_merge(image, basefs) diff --git a/src/vclustermgr.py b/src/vclustermgr.py index 9e23e11..8d813c0 100755 --- a/src/vclustermgr.py +++ b/src/vclustermgr.py @@ -51,7 +51,34 @@ class VclusterMgr(object): logger.info ("recovering cluster:%s for user:%s ..." % (cluster, user)) self.recover_cluster(cluster, user) logger.info("recovered all vclusters for all users") + + def mount_allclusters(self): + logger.info("mounting all vclusters for all users...") + usersdir = self.fspath+"/global/users/" + for user in os.listdir(usersdir): + for cluster in self.list_clusters(user)[1]: + logger.info ("mounting cluster:%s for user:%s ..." % (cluster, user)) + self.mount_cluster(cluster, user) + logger.info("mounted all vclusters for all users") + def stop_allclusters(self): + logger.info("stopping all vclusters for all users...") + usersdir = self.fspath+"/global/users/" + for user in os.listdir(usersdir): + for cluster in self.list_clusters(user)[1]: + logger.info ("stopping cluster:%s for user:%s ..." % (cluster, user)) + self.stop_cluster(cluster, user) + logger.info("stopped all vclusters for all users") + + def detach_allclusters(self): + logger.info("detaching all vclusters for all users...") + usersdir = self.fspath+"/global/users/" + for user in os.listdir(usersdir): + for cluster in self.list_clusters(user)[1]: + logger.info ("detaching cluster:%s for user:%s ..." % (cluster, user)) + self.detach_cluster(cluster, user) + logger.info("detached all vclusters for all users") + def create_cluster(self, clustername, username, image, user_info, setting): if self.is_cluster(clustername, username): return [False, "cluster:%s already exists" % clustername] @@ -325,6 +352,15 @@ class VclusterMgr(object): infofile.close() return [True, "start cluster"] + def mount_cluster(self, clustername, username): + [status, info] = self.get_clusterinfo(clustername, username) + if not status: + return [False, "cluster not found"] + for container in info['containers']: + worker = self.nodemgr.ip_to_rpc(container['host']) + worker.mount_container(container['containername']) + return [True, "mount cluster"] + def recover_cluster(self, clustername, username): [status, info] = self.get_clusterinfo(clustername, username) if not status: @@ -362,6 +398,17 @@ class VclusterMgr(object): infofile.write(json.dumps(info)) infofile.close() return [True, "stop cluster"] + + def detach_cluster(self, clustername, username): + [status, info] = self.get_clusterinfo(clustername, username) + if not status: + return [False, "cluster not found"] + if info['status'] == 'running': + return [False, 'cluster is running, please stop it first'] + for container in info['containers']: + worker = self.nodemgr.ip_to_rpc(container['host']) + worker.detach_container(container['containername']) + return [True, "detach cluster"] def list_clusters(self, user): if not os.path.exists(self.fspath+"/global/users/"+user+"/clusters"): diff --git a/src/worker.py b/src/worker.py index ff69b9a..fcfd8e1 100755 --- a/src/worker.py +++ b/src/worker.py @@ -182,7 +182,7 @@ class Worker(object): if status: # master has know the worker so we start send heartbeat package if value=='ok': - self.etcd.setkey("machines/runnodes/"+self.addr, "ok", ttl = 2) + self.etcd.setkey("machines/runnodes/"+self.addr, "ok", ttl = 60) else: logger.error("get key %s failed, master crashed or initialized. restart worker please." % self.addr) sys.exit(1) diff --git a/web/templates/admin.html b/web/templates/admin.html index c5a351d..f21e716 100644 --- a/web/templates/admin.html +++ b/web/templates/admin.html @@ -201,8 +201,60 @@ - -
+
+
+
+

Update Base Image

+
+ + +
+
+
+
+ + + + + + + + + + + {% for image in root_image %} + + + + + + {% endfor %} + +
ImageNameCreateTimeDescriptionOperation
{{image['name']}}{{image['time']}}{{image['description']}} + +
+
+
+
+
diff --git a/web/web.py b/web/web.py index b2c851e..f583a60 100755 --- a/web/web.py +++ b/web/web.py @@ -235,6 +235,12 @@ def deleteImage(image): deleteImageView.image = image return deleteImageView.as_view() +@app.route("/image/updatebase//", methods=['GET']) +@login_required +def updatebaseImage(image): + updatebaseImageView.image = image + return updatebaseImageView.as_view() + @app.route("/hosts/", methods=['GET']) @administration_required def hosts(): diff --git a/web/webViews/admin.py b/web/webViews/admin.py index 8671ab2..2da3309 100644 --- a/web/webViews/admin.py +++ b/web/webViews/admin.py @@ -14,8 +14,9 @@ class adminView(normalView): quotas = result["quotas"] defaultgroup = result["default"] parms = dockletRequest.post('/system/parmList/') + rootimage = dockletRequest.post('/image/list/').get('images') lxcsetting = dockletRequest.post('/user/lxcsettingList/')['data'] - return self.render(self.template_path, groups = groups, quotas = quotas, defaultgroup = defaultgroup, parms = parms, lxcsetting = lxcsetting) + return self.render(self.template_path, groups = groups, quotas = quotas, defaultgroup = defaultgroup, parms = parms, lxcsetting = lxcsetting, root_image = rootimage['private']) class groupaddView(normalView): @classmethod @@ -95,4 +96,11 @@ class historydelView(normalView): dockletRequest.post('/system/historydel/', request.form) return redirect('/admin/') - +class updatebaseImageView(normalView): + @classmethod + def get(self): + data = { + "image": self.image + } + dockletRequest.post('/image/updatebase/', data) + return redirect("/admin/")