adding scale module

src/cloudmgr.py

@@ -0,0 +1,167 @@
+#!/usr/bin/python3
+from io import StringIO
+import os,sys,subprocess,time,re,datetime,threading,random
+from model import db, Image
+from deploy import *
+
+from log import logger
+import env
+import requests
+
+fspath = env.getenv('FS_PREFIX')
+
+class CludAccountMgr():
+    def cloud_account_query(*args, **kwargs):
+        try:
+            accountfile = open(fspath+"/global/sys/cloudaccount", 'r')
+            account = json.loads(accountfile.read())
+            accountfile.close()
+        except:
+            account = {}
+            account['cloud'] = env.getenv('CLOUD') 
+            account['accesskey'] = ""
+            account['accesssecret'] = ""
+        return {"success": 'true', 'accounts':account}
+
+    def cloud_account_modify(*args, **kwargs):
+        form = kwargs.get('form')
+        account = {}
+        account['cloud'] = form['cloud']
+        account['accesskey'] = form['accesskey']
+        account['accesssecret'] = form['accesssecret']
+        accountfile = open(fspath+"/global/sys/cloudaccount", 'w')
+        accountfile.write(json.dumps(account))
+        accountfile.close()
+        return {"success": "true"}
+
+
+class AliyunMgr():
+    def __init__(self):
+        self.AcsClient = __import__('aliyunsdkcore.client')
+        self.Request = __import__('aliyunsdkecs.request.v20140526')
+
+    def loadClient(self):
+        try:
+            accountfile = open(fspath+"/global/sys/cloudaccount", 'r')
+            account = json.loads(accountfile.read())
+            accountfile.close()
+            self.clt = self.AcsClient.AcsClient(account['accesskey'],account['accesssecret'],'cn-shanghai')
+            logger.info("load CLT of Aliyun success")
+            return True
+        except:
+            logger.error("account file not existed, can not load CLT")
+            return False
+    
+    def createInstance(self,password):
+        request = self.Request.CreateInstanceRequest.CreateInstanceRequest()
+        request.set_accept_format('json')
+        request.add_query_param('RegionId', 'cn-shanghai')
+        request.add_query_param('ImageId', 'ubuntu_16_0402_64_20G_alibase_20170818.vhd')
+        request.add_query_param('InternetMaxBandwidthOut', 1)
+        request.add_query_param('InstanceName', 'docklet_tmp_worker')
+        request.add_query_param('HostName', 'worker-tmp')
+        request.add_query_param('SystemDisk.Size', 500)
+        request.add_query_param('InstanceType', 'ecs.xn4.small')
+        request.add_query_param('Password', password)
+        response = self.clt.do_action_with_exception(request)
+        logger.info(response)
+    
+        # 获取实例ID
+        instanceid=json.loads(bytes.decode(response))['InstanceId']
+        return instanceid
+        
+    # 启动ECS
+    def startInstance(self, instanceid):
+        request = self.Request.StartInstanceRequest.StartInstanceRequest()
+        request.set_accept_format('json')
+        request.add_query_param('InstanceId', instanceid)
+        response = self.clt.do_action_with_exception(request)
+        logger.info(response)
+        
+    
+    # 创建EIP
+    def createEIP(self):
+        request = self.Request.AllocateEipAddressRequest.AllocateEipAddressRequest()
+        request.set_accept_format('json')
+        request.add_query_param('RegionId', 'cn-shanghai')
+        response = self.clt.do_action_with_exception(request)
+        logger.info(response)
+    
+        response=json.loads(bytes.decode(response))
+        eipid=response['AllocationId']
+        eipaddr=response['EipAddress']
+
+        return [eipid, eipaddr]
+
+
+    # 绑定EIP
+    def associateEIP(self, instanceid, eipid):
+        request = self.Request.AssociateEipAddressRequest.AssociateEipAddressRequest()
+        request.set_accept_format('json')
+        request.add_query_param('AllocationId', eipid)
+        request.add_query_param('InstanceId', instanceid)
+        response = self.clt.do_action_with_exception(request)
+        logger.info(response)
+
+    
+    def getInnerIP(self, instanceid):
+        request = self.Request.DescribeInstancesRequest.DescribeInstancesRequest()
+        request.set_accept_format('json')
+        response = self.clt.do_action_with_exception(request)
+        instances = json.loads(bytes.decode(response))['Instances']['Instance']
+        for instance in instances:
+            if instance['InstanceId'] == instanceid:
+                return instance['NetworkInterfaces']['NetworkInterface'][0]['PrimaryIpAddress']
+        return json.loads(bytes.decode(response))['Instances']['Instance'][0]['VpcAttributes']['PrivateIpAddress']['IpAddress'][0]
+
+    def isStarted(self, instanceids):
+        request = self.Request.DescribeInstancesRequest.DescribeInstancesRequest()
+        request.set_accept_format('json')
+        response = self.clt.do_action_with_exception(request)
+        instances = json.loads(bytes.decode(response))['Instances']['Instance']
+        for instance in instances:
+            if instance['InstanceId'] in instanceids:
+                if not instance['Status'] == "Running":
+                    return False
+        return True
+
+    def rentServers(self):
+        instanceids=[]
+        eipids=[]
+        eipaddrs=[]
+        for i in range(int(number)):
+            instanceids.append(self.createInstance(password))
+            time.sleep(2)
+        time.sleep(10)
+        for i in range(int(number)):
+            [eipid,eipaddr]=self.createEIP()
+            eipids.append(eipid)
+            eipaddrs.append(eipaddr)
+            time.sleep(2)
+        masterip=env.getenv('ETCD').split(':')[0]
+        for i in range(int(number)):
+            self.associateEIP(instanceids[i],eipids[i])
+            time.sleep(2)
+        time.sleep(5)
+        for instanceid in instanceids:
+            self.startInstance(instanceid)
+            time.sleep(2)
+        time.sleep(10)
+        while not self.isStarted(instanceids):
+            time.sleep(10)
+        time.sleep(5)
+        return [masterip, eipaddrs]
+
+    def addNodes(self,number=1,password="Unias1616"):
+        if not loadClient():
+            return False
+        [masterip, eipaddrs] = self.rentServers(number,password)
+        threads = []
+        for eip in eipaddrs:
+            thread = threading.Thread(target = deploy, args=(eip,masterip,'root',password))
+            thread.setDaemon(True)
+            thread.start()
+            threads.append(thread)
+        for thread in threads:
+            thread.join()
+        return True

src/deploy.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+
+#实例ip调用describe api获取
+import paramiko, time
+from log import logger
+import env
+
+def myexec(ssh,command):
+    stdin,stdout,stderr = ssh.exec_command(command)
+    endtime = time.time() + 300
+    while not stdout.channel.eof_received:
+        time.sleep(2)
+        if time.time() > endtime:
+            stdout.channel.close()
+            logger.error(command + ": fail")
+#    for line in stdout.readlines():
+#        if line is None:
+#            time.sleep(5)
+#        else:
+#            print(line)
+
+#上传deploy脚本
+def deploy(ipaddr,masterip,account,password):
+    while True:
+        try:
+            transport = paramiko.Transport((ipaddr,22))
+            transport.connect(username=account,password=password)
+            break
+        except Exception as e:
+            time.sleep(2)
+            pass
+    sftp = paramiko.SFTPClient.from_transport(transport)
+
+    fspath = env.getenv('FS_PREFIX')
+    sftp.put('/home/zhong/docklet-deploy.sh','/root/docklet-deploy.sh')
+    sftp.put('/home/zhong/docklet-deploy.sh','/root/docklet-deploy.sh')
+    transport.close()
+    
+    #执行deploy脚本
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    while True:
+        try:
+            ssh.connect(ipaddr, username = account, password = password, timeout = 300)
+            break
+        except Exception as e:
+            time.sleep(2)
+            pass
+    #这行日后可以删掉
+    myexec(ssh,'chmod +x /root/docklet-deploy.sh')
+    myexec(ssh,"sed -i 's/%MASTERIP%/" + masterip + "/g' /root/docklet-deploy.sh")
+    myexec(ssh,'/root/docklet-deploy.sh ' + ipaddr)
+    myexec(ssh,'mount -t glusterfs ' + masterip + ':docklet /opt/docklet/global/')
+    myexec(ssh,'/home/docklet/bin/docklet-worker start')
+    ssh.close()
+    return

src/httprest.py

@@ -438,6 +438,39 @@ def copytarget_image(user, beans, form):
         return json.dumps({'success':'false', 'message':str(e)})
     return json.dumps({'success':'true', 'action':'copy image to target.'})
 
+@app.route("/cloud/account/query/", methods=['POST'])
+@login_required
+def query_account_cloud(cur_user, user, form):
+    global G_usermgr
+    logger.info("handle request: cloud/account/query/")
+    result = G_usermgr.cloud_account_query(cur_user = cur_user)
+    return json.dumps(result)
+
+@app.route("/cloud/account/add/", methods=['POST'])
+@login_required
+def add_account_cloud(cur_user, user, form):
+    global G_usermgr
+    logger.info("handle request: cloud/account/add/")
+    result = G_usermgr.cloud_account_add(cur_user = cur_user, form = form)
+    return json.dumps(result)
+
+@app.route("/cloud/account/delete/", methods=['POST'])
+@login_required
+def del_account_cloud(cur_user, user, form):
+    global G_usermgr
+    logger.info("handle request: cloud/account/delete/")
+    result = G_usermgr.cloud_account_del(cur_user = cur_user, form = form)
+    return json.dumps(result)
+
+@app.route("/cloud/account/modify/", methods=['POST'])
+@login_required
+def modify_account_cloud(cur_user, user, form):
+    global G_usermgr
+    logger.info("handle request: cloud/account/modify/")
+    result = G_usermgr.cloud_account_modify(cur_user = cur_user, form = form)
+    return json.dumps(result)
+
+
 @app.route("/addproxy/", methods=['POST'])
 @login_required
 def addproxy(user, beans, form):

src/userManager.py

@@ -968,62 +968,6 @@ class userManager:
         lxcsettingfile.close()
         return {"success": 'true'}
 
-    @administration_required
-    def cloud_account_query(*args, **kwargs):
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'r')
-        account = json.loads(accountfile.read())
-        accountfile.close()
-        return {"success": 'true', 'accounts':account}
-
-    @administration_required
-    def cloud_account_add(*args, **kwargs):
-        form = kwargs.get('form')
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'r')
-        account = json.loads(accountfile.read())
-        accountfile.close()
-        account.append(
-                { 'cloudname' : form['cloudname'],
-                    'username' : form['username'],
-                    'password' : form['password'],
-                })
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'w')
-        accountfile.write(json.dumps(account))
-        accountfile.close()
-        return {"success": 'true'}
-
-    @administration_required
-    def cloud_account_del(*args, **kwargs):
-        form = kwargs.get('form')
-        cloudname = form['cloudname']
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'r')
-        account = json.loads(accountfile.read())
-        accountfile.close()
-        for acc in account:
-            if acc['cloudname'] == cloudname:
-                account.remove(acc)
-                break
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'w')
-        accountfile.write(json.dumps(account))
-        accountfile.close()
-        return {"success": 'true'}
-
-    @administration_required
-    def cloud_account_modify(*args, **kwargs):
-        form = kwargs.get('form')
-        cloudname = form['cloudname']
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'r')
-        account = json.loads(accountfile.read())
-        accountfile.close()
-        for acc in account:
-            if acc['cloudname'] == cloudname:
-                acc['username'] = form['username']
-                acc['password'] = form['password']
-                break
-        accountfile = open(fspath+"/global/sys/cloudaccount", 'w')
-        accountfile.write(json.dumps(account))
-        accountfile.close()
-        return {"success": "true"}
-
 
     def queryForDisplay(*args, **kwargs):
         '''

tools/docklet-deploy.sh

@@ -1,9 +1,3 @@
-if [ $# -lt 1 ]; then
-    echo "please input master ip";
-    exit 1;
-fi
-MASTER_IP=$1
-
 #配置apt源
 #echo "deb http://mirrors.ustc.edu.cn/ubuntu/ xenial main restricted universe multiverse
 #deb http://mirrors.ustc.edu.cn/ubuntu/ xenial-security main restricted universe multiverse
@@ -14,40 +8,44 @@ MASTER_IP=$1
 #更新apt
 apt-get update
 
-#更新hosts
-echo "$MASTER_IP docklet-master" >> /etc/hosts
 
 #下载git包
-apt-get -y install git
+apt-get install -y git
 
 #下载docklet源码
 git clone http://github.com/unias/docklet.git /home/docklet
 
 #运行prepare.sh
+sed -i '61s/^/#&/g' /home/docklet/prepare.sh
+sed -i '62s/^/#&/g' /home/docklet/prepare.sh
 /home/docklet/prepare.sh
 
 #挂载global目录,通过gluster方式
-mount -t glusterfs docklet-master:/docklet /opt/docklet/global
+#mount -t glusterfs docklet-master:/docklet /opt/docklet/global
 
 #下载base镜像
-wget http://docklet.unias.org/images/basefs-0.11.tar.bz2 -P /opt/local/temp
+#wget http://docklet.unias.org/images/basefs-0.11.tar.bz2 -P /opt/local/temp
 
 #解压镜像
-tar -zxvf /opt/local/temp/basefs-0.11.tar.bz2 -C /opt/local/
+#mkdir -p /opt/docklet/local
+#mkdir -p /opt/docklet/global
+#tar -jxvf /root/basefs-0.11.tar.bz2 -C /opt/docklet/local/
 
 #获得docklet.conf
 cp /home/docklet/conf/docklet.conf.template /home/docklet/conf/docklet.conf
+cp /home/docklet/web/templates/home.template /home/docklet/web/templates/home.html
 
 #获得网卡名称
 NETWORK_DEVICE=`route | grep default | awk {'print $8'};`
 
 #更改配置文件
-echo "DISKPOOL_SIZE=10000
+echo "DISKPOOL_SIZE=20000
-ETCD=$MASTER_IP:2379
+ETCD=%MASTERIP%:2379
 NETWORK_DEVICE=$NETWORK_DEVICE
-PORTAL_URL=http://iwork.pku.edu.cn
+PROXY_PORT=8000
-PROXY_PORT=80" >> /home/docklet/conf/docklet.conf
+NGINX_PORT=80" >> /home/docklet/conf/docklet.conf
 
 #启动worker
-/home/docklet/bin/docklet-worker start
+#/home/docklet/bin/docklet-supermaster init
+#/home/docklet/bin/docklet-worker start
 exit 0

user/user.py

@@ -164,38 +164,6 @@ def auth_token(cur_user, user, form):
      logger.info("auth success")
      return req
 
-@app.route("/cloud/account/query/", methods=['POST'])
-@login_required
-def query_account_cloud(cur_user, user, form):
-    global G_usermgr
-    logger.info("handle request: cloud/account/query/")
-    result = G_usermgr.cloud_account_query(cur_user = cur_user)
-    return json.dumps(result)
-
-@app.route("/cloud/account/add/", methods=['POST'])
-@login_required
-def add_account_cloud(cur_user, user, form):
-    global G_usermgr
-    logger.info("handle request: cloud/account/add/")
-    result = G_usermgr.cloud_account_add(cur_user = cur_user, form = form)
-    return json.dumps(result)
-
-@app.route("/cloud/account/delete/", methods=['POST'])
-@login_required
-def del_account_cloud(cur_user, user, form):
-    global G_usermgr
-    logger.info("handle request: cloud/account/delete/")
-    result = G_usermgr.cloud_account_del(cur_user = cur_user, form = form)
-    return json.dumps(result)
-
-@app.route("/cloud/account/modify/", methods=['POST'])
-@login_required
-def modify_account_cloud(cur_user, user, form):
-    global G_usermgr
-    logger.info("handle request: cloud/account/modify/")
-    result = G_usermgr.cloud_account_modify(cur_user = cur_user, form = form)
-    return json.dumps(result)
-
 
 @app.route("/user/modify/", methods=['POST'])
 @login_required

web/webViews/cloud.py

@@ -9,7 +9,7 @@ class cloudView(normalView):
 
     @classmethod
     def post(self):
-        accounts = dockletRequest.post('/cloud/account/query/').get('accounts',[])
+        accounts = dockletRequest.post_to_all('/cloud/account/query/')
         return self.render(self.template_path, accounts = accounts)
 
     @classmethod

web/webViews/dockletrequest.py

@@ -38,7 +38,6 @@ class dockletRequest():
                 'user',
                 'beans',
                 'notification',
-                'cloud',
                 'settings'
                 }
         if ":" not in endpoint: