Merge branch 'backend_dev' of gitlab2.kylin.com:kylin-desktop/update-manager-group/kylin-system-updater into backend_dev

backend/SystemUpdater/UpdateManager.py

@@ -1,6 +1,7 @@
 # UpdateManager.py
 # -*- Mode: Python; indent-tabs-mode: nil; tab-width: 4; coding: utf-8 -*-
-
+import apt
+import fcntl
 import os
 from apt import Cache
 import sys
@@ -25,7 +26,7 @@ from gettext import gettext as _
 from SystemUpdater.backend import DownloadBackend as downb
 import apt_pkg
 from SystemUpdater.Core.UpdaterConfigParser import UpgradeConfig
-from SystemUpdater.Core.utils import get_broken_details,get_lis_from_cache
+from SystemUpdater.Core.utils import country_mirror, get_broken_details,get_lis_from_cache
 
 import subprocess
 
@@ -152,6 +153,7 @@ class UpdateManager():
         except Exception as e:
             logging.error(e)
 
+    
     # 进行本地deb包安装的操作
     def start_deb_install(self, deb_path = "", _check_local_dep = False, _auto_satisfy = False):
         # _check_local_dep : 是否查询本地依赖
@@ -160,20 +162,17 @@ class UpdateManager():
         desc = ''
         absolute_path, debname = os.path.split(deb_path)
         try:
-            # 是否有破损的包
-            deb_cache = Cache()
-            broken_count = deb_cache._depcache.broken_count
-            _need_downgrade = self._check_downgrade(deb_cache, debname)
-            if broken_count > 0 or _need_downgrade == True:
-            # 走 dpkg 安装流程，说明本地apt环境已经损坏,or downgrade
-                dep_satisfy, header, desc = self._deb_install(deb_cache, deb_path, _check_local_dep)
-                if dep_satisfy:
+            deb_cache, ins = self._suit_install_mode(deb_path)
+            if self._is_broken > 0 or not self.cacheSatisfy or self._need_downgrade:
+            # 走 dpkg 安装流程，说明本地apt环境已经损坏,or dep not satisfied or need downgrade
+                dep_satisfy, header, desc = self._deb_install(deb_cache, deb_path, _check_local_dep, ins)
+                if dep_satisfy: 
                     self.dbusController.InstalldebFinished(True, header, desc)
                 else:
                     self.dbusController.InstalldebFinished(False, header, desc)
             else:
             # apt 安装流程
-                dep_satisfy, header, desc = self._attempt_depends(deb_cache, deb_path, _check_local_dep,_auto_satisfy)
+                dep_satisfy, header, desc = self._attempt_depends(deb_cache, deb_path, _check_local_dep,_auto_satisfy, ins)
                 if dep_satisfy:
                     try:
                         install_backend = get_backend(self, InstallBackend.ACTION_INSTALL_DEB)
@@ -184,6 +183,7 @@ class UpdateManager():
                     self.dbusController.InstalldebFinished(False, header, desc)
         except Exception as e:
             logging.info(str(e))
+            self.dbusController.InstalldebFinished(False, str(e), desc)
 
     #进行升级的操作
     def start_install(self,upgrade_mode,is_install = False,partial_upgrade_list = []):
@@ -456,7 +456,7 @@ class UpdateManager():
             return  UpdateManagerDbusController(self, bus_name)
 
     # 是否查找本地依赖
-    def _attempt_depends(self,deb_cache, deb_path,_check_local_dep,_auto_satisfy):  
+    def _attempt_depends(self,deb_cache, deb_path,_check_local_dep,_auto_satisfy, _install):  
         depends_list = []
         depends_pkg = []
         satisfy_list = []
@@ -465,22 +465,11 @@ class UpdateManager():
         error_string = ''
         error_desc = ''
         absolute_path, debname = os.path.split(deb_path)
-        try:
-            deb = DebPackage(deb_path, deb_cache)
-            deb.check()
-            (install, remove, unauth) = deb.required_changes
-        except Exception as e:
-            logging.error(str(e))
-            return False,str(e), error_desc
         # 依赖不满足的情况
-        if len(install) > 0:
+        if len(_install) > 0:
             if _check_local_dep: #查找本地
-                if remove:
-                    logging.error("Need uninstall: %s.",str(remove))
-                    error_string = "Installing "+str(debname.split("_")[0])+" requires uninstalling: "+str(remove)
-                    logging.error(error_string)
                 # 需要查找本地依赖
-                elif len(install) > 0:
+                if len(_install) > 0:
                     for pkg in deb_cache:
                         if pkg.marked_install and pkg.name != str(debname.split("_")[0]):
                             depends_pkg.append(pkg)
@@ -501,7 +490,7 @@ class UpdateManager():
                         elif not _auto_satisfy:
                             _local_satisfy = False
                             error_string = str(debname.split("_")[0])+" dependency is not satisfied. "
-                            error_desc   = ",".join(install)
+                            error_desc   = ",".join(_install)
                             logging.error(error_string+ error_desc)
                     else:
                         #将应用包与依赖包拷贝至archive目录安装
@@ -517,16 +506,16 @@ class UpdateManager():
                 return _local_satisfy,error_string,error_desc
             elif not _check_local_dep and _auto_satisfy:
                 _local_satisfy = True
-                if install:
+                if _install:
                     error_string = str(debname.split("_")[0])+" dependency is not satisfied, will download. "
-                    error_desc   = ",".join(install)
+                    error_desc   = ",".join(_install)
                     logging.error(error_string+error_desc)
                     return _local_satisfy,error_string,error_desc
             elif not _check_local_dep and not _auto_satisfy:
                 _local_satisfy = False
-                if install:
+                if _install:
                     error_string = str(debname.split("_")[0])+" dependency is not satisfied. "
-                    error_desc   = ",".join(install)
+                    error_desc   = ",".join(_install)
                     logging.error(error_string+error_desc)
                     return _local_satisfy,error_string,error_desc
         # 依赖满足
@@ -536,7 +525,7 @@ class UpdateManager():
             error_desc=''
             return _local_satisfy,error_string,error_desc
     
-    def _deb_install(self, deb_cache, deb_path, _check_local_dep):
+    def _deb_install(self, deb_cache, deb_path, _check_local_dep, _install):
         depends_list = []
         depends_pkg = []
         satisfy_list = []
@@ -546,34 +535,34 @@ class UpdateManager():
         header = ''
         desc = ''
         absolute_path, debname = os.path.split(deb_path)
-        try:
-            deb = DebPackage(deb_path, deb_cache)
-            deb.check()
-            (install, remove, unauth) = deb.required_changes
-        except Exception as e:
-            logging.error(str(e))
-            return False,str(e), desc
-         # 依赖不满足的情况
-        if len(install) > 0:
+        # 依赖不满足的情况
+        if len(_install) > 0 or len(self.noSatisfyList) > 0:
             if _check_local_dep: #查找本地
                 # 需要查找本地依赖
-                if len(install) > 0:
+                if len(self.noSatisfyList) > 0:
+                    noSatisfyList = [ str(nS[self.noSatisfyList.index(nS)][0]+'_'+nS[self.noSatisfyList.index(nS)][1]) for nS in self.noSatisfyList]
+                if len(_install) > 0:
                     for pkg in deb_cache:
                         if pkg.marked_install and pkg.name != str(debname.split("_")[0]):
                             depends_pkg.append(pkg)
                         elif pkg.marked_upgrade and pkg.name != str(debname.split("_")[0]):
                             depends_pkg.append(pkg)
-                if len(depends_pkg)>0: #查找本地deb包
+                if len(depends_pkg)>0 or len(noSatisfyList)>0: #查找本地deb包
                     depends_list = [debfile for debfile in os.listdir(absolute_path) if debfile.endswith(".deb")]
                     for depends in depends_pkg:
                         for debfile in depends_list:
                             if depends.name in debfile and depends.candidate.version in debfile:
                                 depends_count += 1
                                 satisfy_list.append(debfile)
-                    if depends_count < len(depends_pkg):
+                    for depends in noSatisfyList:
+                        for debfile in depends_list:
+                            if depends.split('_')[0] == debfile.split('_')[0] and depends.split('_')[1] == debfile.split('_')[1] and debfile not in satisfy_list:
+                                depends_count += 1
+                                satisfy_list.append(debfile)
+                    if depends_count < len(noSatisfyList):
                         #本地依赖不满足
                         _status = False
-                        error_string = str(debname.split("_")[0])+" dependency is not satisfied. "+",".join(install)
+                        error_string = str(debname.split("_")[0])+" dependency is not satisfied. "+",".join(noSatisfyList)
                         logging.error(error_string)
                         header = error_string
                         return False,header, desc
@@ -584,15 +573,16 @@ class UpdateManager():
                         for satisfy in satisfy_list:
                             try:
                                 deb = DebPackage(os.path.join(absolute_path,satisfy))
-                                ret = deb.install()
+                                iprogress = LogInstallProgress(deb_path)
+                                ret = deb.install(install_progress=iprogress)
                             except Exception as e:
                                 logging.error(e)
                                 return False, str(e), desc
                         return True, header, desc
             elif not _check_local_dep:
                 _status = False
-                if install:
-                    error_string = str(debname.split("_")[0])+" dependency is not satisfied. "+",".join(install)
+                if _install:
+                    error_string = str(debname.split("_")[0])+" dependency is not satisfied. "+",".join(_install)
                     logging.error(error_string)
                     header = error_string
                     return False,header, desc
@@ -600,22 +590,204 @@ class UpdateManager():
         else:
             try:
                 deb = DebPackage(deb_path)
-                ret = deb.install()
+                iprogress = LogInstallProgress(deb_path)
+                ret = deb.install(install_progress=iprogress)
+                if ret != 0:
+                   return False, iprogress.errormsg, desc
             except Exception as e:
                 logging.error(e)
                 return False, str(e), desc
             return True, header, desc
-    
-    def _check_downgrade(self, deb_cache, debname):
-        if deb_cache == None or debname == "":
-            return
-        pkgname = debname.split('_')[0]
-        pkg_version = debname.split('_')[1]
+
+    def _suit_install_mode(self, deb_path):
+        self._is_broken = False
+        self.cacheSatisfy = False
+        absolute_path, debname = os.path.split(deb_path)
+        # 检查本地破损
         try:
-            pkg = deb_cache[pkgname]
-            if pkg.is_installed and pkg.installed.source_version > pkg_version:
-                return True
-            else:
-                return False
+            logging.info("Install package, open cache")
+            deb_cache = Cache()
+            logging.info("Install package, check broken")
+            broken_count = deb_cache._depcache.broken_count
+            deb = DebPackage(deb_path, deb_cache)
+            deb.check(allow_downgrade=True)
+            logging.info("Install package, required changes")
+            (install, remove, unauth) = deb.required_changes    # need in cach
+            if broken_count > 0:
+                self._is_broken = True
+            else :
+                self._is_broken = False
+        except KeyError:
+            pass
         except Exception as e:
-            logging.error(str(e))
+            logging.error(str(e))
+        # 需要降级
+        try:
+            pkg = deb_cache[debname.split('_')[0]]
+            if pkg.is_installed and pkg.installed.source_version > debname.split('_')[1]:
+                logging.info("Downgrade deb %s, from %s to %s.",\
+                                debname.split('_')[0], pkg.installed.source_version,debname.split('_')[1])
+                self._need_downgrade = True
+            else:
+                self._need_downgrade = False
+        except Exception as e:
+            logging.error(str(e))
+            self._need_downgrade = False
+        # 不满足的依赖列表
+        depends = deb.depends
+        self.noSatisfyList = self._gen_noSatisfyList(depends, deb_cache)
+        _list = []
+        # cache是否满足
+        if len(install) == 0 and len(self.noSatisfyList) == 0:
+            self.cacheSatisfy = True
+        else:
+            for ns in self.noSatisfyList:
+                for or_group in ns:
+                    for pkg in install:
+                        if pkg == or_group[0]:
+                            _list.append(ns)
+            if len(_list) == len(self.noSatisfyList):
+                self.cacheSatisfy = True
+            else:
+                self.cacheSatisfy = False
+        logging.info("Cache satisfy is %r.",self.cacheSatisfy)
+        return deb_cache, install
+    
+    def _gen_noSatisfyList(self, depends, deb_cache):
+        _noSatisfyList = []
+        for or_group in depends:
+            for deb in or_group:
+                debname,ver,oper = deb
+                try:
+                    pkg = deb_cache[debname]
+                    if not pkg.installed or not apt_pkg.check_dep(pkg.installed.source_version, oper, ver):
+                        _noSatisfyList.append(or_group)
+                except Exception as e:
+                    logging.error(str(e))
+        return _noSatisfyList
+                    
+
+class LogInstallProgress(apt.progress.base.InstallProgress):
+    """ Install progress that writes to self.progress_log
+        (/var/run/unattended-upgrades.progress by default)
+    """
+    def __init__(self,file):
+        # type: (str) -> None
+        apt.progress.base.InstallProgress.__init__(self)
+        self.output_logfd = None  # type: int
+        self.filename=file
+        self.error_pkg=""
+        self.errormsg=""
+
+    def error(self,pkg, errormsg):
+        logging.info(("errormsg:%s"),errormsg)
+        self.error_pkg=self.filename
+        self.errormsg = errormsg
+        global error_status
+        global error_msg
+        error_status=1
+        error_msg=errormsg
+
+    def status_change(self, pkg, percent, status):
+        # type: (str, float, str) -> None
+        logging.info(("pkg:%s,percent:%s,status:%s"),pkg,percent,status)
+        with open(self.progress_log, "w") as f:
+            f.write(("%s")%percent)
+            f.write(_("当前进度: %s ，正在安装：%s，当前状态：%s") % (percent, pkg,status))
+            f.write(_("Progress: %s %% (%s)") % (percent, pkg))
+
+    def _fixup_fds(self):
+        # () -> None
+        required_fds = [0, 1, 2,  # stdin, stdout, stderr
+                        self.writefd,
+                        self.write_stream.fileno(),
+                        self.statusfd,
+                        self.status_stream.fileno()
+                        ]
+        # ensure that our required fds close on exec
+        for fd in required_fds[3:]:
+            old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+            fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
+        # close all fds
+        try:
+            # PEP-446 implemented in Python 3.4 made all descriptors
+            # CLOEXEC, but we need to be able to pass writefd to dpkg
+            # when we spawn it
+            os.set_inheritable(self.writefd, True)
+        except AttributeError:  # if we don't have os.set_inheritable()
+            pass
+        proc_fd = "/proc/self/fd"
+        if os.path.exists(proc_fd):
+            error_count = 0
+            for fdname in os.listdir(proc_fd):
+                try:
+                    fd = int(fdname)
+                except Exception:
+                    print("ERROR: can not get fd for %s" % fdname)
+                if fd in required_fds:
+                    continue
+                try:
+                    os.close(fd)
+                    # print("closed: ", fd)
+                except OSError as e:
+                    # there will be one fd that can not be closed
+                    # as its the fd from pythons internal diropen()
+                    # so its ok to ignore one close error
+                    error_count += 1
+                    if error_count > 1:
+                        print("ERROR: os.close(%s): %s" % (fd, e))
+
+    def _redirect_stdin(self):
+        # type: () -> None
+        REDIRECT_INPUT = os.devnull
+        fd = os.open(REDIRECT_INPUT, os.O_RDWR)
+        os.dup2(fd, 0)
+
+    def _redirect_output(self):
+        # type: () -> None
+        # do not create log in dry-run mode, just output to stdout/stderr
+        if not apt_pkg.config.find_b("Debug::pkgDPkgPM", False):
+            logfd = self._get_logfile_dpkg_fd()
+            os.dup2(logfd, 1)
+            os.dup2(logfd, 2)
+
+    def _get_logfile_dpkg_fd(self):
+        # type: () -> int
+        logfd = os.open(
+            "/var/log/kylin-system-updater/kylin-system-updater.log.1", os.O_RDWR | os.O_APPEND | os.O_CREAT, 0o640)
+        try:
+            import grp
+            adm_gid = grp.getgrnam("adm").gr_gid
+            os.fchown(logfd, 0, adm_gid)
+        except (KeyError, OSError):
+            pass
+        return logfd
+
+    def update_interface(self):
+        # type: () -> None
+        # call super class first
+        apt.progress.base.InstallProgress.update_interface(self)
+
+    def _log_in_dpkg_log(self, msg):
+        # type: (str) -> None
+        logfd = self._get_logfile_dpkg_fd()
+        os.write(logfd, msg.encode("utf-8"))
+        os.close(logfd)
+
+    def finish_update(self):
+        # type: () -> None
+        # logging.info("finished")
+        # if error_status == 1:
+        #     os._exit(1)
+        pass
+
+    def fork(self):
+        # type: () -> int
+        pid = os.fork()
+        if pid == 0:
+            self._fixup_fds()
+            self._redirect_stdin()
+            self._redirect_output()
+        return pid
+    
+    

backend/SystemUpdater/UpdateManagerDbus.py

@@ -326,6 +326,7 @@ class UpdateManagerDbusController(dbus.service.Object):
             deb_path = str(path)
             logging.info(COLORMETHOR_PREFIX+'method'+COLORLOG_SUFFIX+' InstallDebFile and check_local_dep:%r, auto_satisfy:%r.',\
                                         check_local_dep,auto_satisfy)
+            logging.info("Will install: %s.",path)
             self.parent.start_deb_install(deb_path, _check_local_dep, _auto_satisfy)
             return True
         except Exception as e:

backend/data/com.kylin.systemupgrade.service

@@ -0,0 +1,4 @@
+[D-BUS Service]
+Name=com.kylin.systemupgrade
+Exec=/usr/share/kylin-system-updater/kylin-system-updater
+User=root

backend/data/kylin-system-updater.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=kylin-system-updater dbus daemon
-StartLimitIntervalSec=0
-
-[Service]
-Type=dbus
-Restart=always
-RestartSec=3
-BusName=com.kylin.systemupgrade
-ExecStart=/usr/share/kylin-system-updater/kylin-system-updater
-
-[Install]
-WantedBy=multi-user.target

debian/kylin-system-updater.install

@@ -1,8 +1,8 @@
 #!/usr/bin/dh-exec
 backend/data/kylin-system-updater.db /usr/share/kylin-system-updater/
 backend/data/30kylin-system-updater /etc/apt/apt.conf.d/
-backend/data/kylin-system-updater.service /usr/lib/systemd/system/
 backend/data/com.kylin.systemupgrade.conf /etc/dbus-1/system.d/
+backend/data/com.kylin.systemupgrade.service /usr/share/dbus-1/system-services/
 backend/kylin-system-updater /usr/share/kylin-system-updater/
 backend/SystemUpdater/*.py /usr/share/kylin-system-updater/SystemUpdater/
 backend/SystemUpdater/backend/*.py /usr/share/kylin-system-updater/SystemUpdater/backend/