From 2debf49d7ee97e9ca2258231b4c43217b69ca2f7 Mon Sep 17 00:00:00 2001 From: Ken Conley Date: Thu, 1 Oct 2009 01:32:07 +0000 Subject: [PATCH] roswtf: #1527 check for invalid stack dependencies. This required beefing up roslib stack libraries --- core/roslib/src/roslib/scriptutil.py | 98 ++++++++++++++++++++---- core/roslib/src/roslib/stack_manifest.py | 13 +--- core/roslib/src/roslib/stacks.py | 4 + tools/roswtf/src/roswtf/__init__.py | 17 ++-- tools/roswtf/src/roswtf/context.py | 37 ++++++++- tools/roswtf/src/roswtf/packages.py | 10 +-- tools/roswtf/src/roswtf/roslaunchwtf.py | 4 +- tools/roswtf/src/roswtf/stacks.py | 93 ++++++++++++++++++++++ 8 files changed, 234 insertions(+), 42 deletions(-) create mode 100644 tools/roswtf/src/roswtf/stacks.py diff --git a/core/roslib/src/roslib/scriptutil.py b/core/roslib/src/roslib/scriptutil.py index ec781347..fb137af0 100644 --- a/core/roslib/src/roslib/scriptutil.py +++ b/core/roslib/src/roslib/scriptutil.py @@ -58,13 +58,18 @@ PRODUCT = 'ros' ## caller ID for master calls where caller ID is not vital _GLOBAL_CALLER_ID = '/script' -## Name resolver for scripts. Supports ROS_NAMESPACE. Does not -## support remapping arguments. -## @param name str: name to resolve -## @param script_name str: name of script. script_name must not -## contain a namespace. -## @return str: resolved name def script_resolve_name(script_name, name): + """ + Name resolver for scripts. Supports ROS_NAMESPACE. Does not + support remapping arguments. + @param name: name to resolve + @type name: str + @param script_name: name of script. script_name must not + contain a namespace. + @type script_name: str + @return: resolved name + @rtype: str + """ if not name: #empty string resolves to namespace return roslib.names.get_ros_namespace() #Check for global name: /foo/name resolves to /foo/name @@ -75,49 +80,110 @@ def script_resolve_name(script_name, name): return ns_join(roslib.names.make_caller_id(script_name), name[1:]) return roslib.names.get_ros_namespace() + name -## @return str: result of executing command (via subprocess). string will be strip()ed. def rospackexec(args): + """ + @return: result of executing rospack command (via subprocess). string will be strip()ed. + @rtype: str + @raise roslib.exceptions.ROSLibException: if rospack command fails + """ + val = (subprocess.Popen(['rospack'] + args, stdout=subprocess.PIPE).communicate()[0] or '').strip() if val.startswith('rospack:'): #rospack error message - raise Exception(val) + raise roslib.exceptions.ROSLibException(val) return val -## @return list: A list of the names of the packages which depend directly on pkg def rospack_depends_on_1(pkg): + """ + @param pkg: package name + @type pkg: str + @return list: A list of the names of the packages which depend directly on pkg + """ return rospackexec(['depends-on1', pkg]).split() -## @return list: A list of the names of the packages which depend on pkg def rospack_depends_on(pkg): + """ + @param pkg: package name + @type pkg: str + @return list: A list of the names of the packages which depend on pkg + """ return rospackexec(['depends-on', pkg]).split() -## @return list: A list of the names of the packages which pkg directly depends on def rospack_depends_1(pkg): + """ + @param pkg: package name + @type pkg: str + @return list: A list of the names of the packages which pkg directly depends on + """ return rospackexec(['deps1', pkg]).split() -## @return list: A list of the names of the packages which pkg depends on def rospack_depends(pkg): + """ + @param pkg: package name + @type pkg: str + @return list: A list of the names of the packages which pkg depends on + """ return rospackexec(['deps', pkg]).split() -## @return list: A list of the names of the packages which provide a plugin for pkg def rospack_plugins(pkg): + """ + @param pkg: package name + @type pkg: str + @return list: A list of the names of the packages which provide a plugin for pkg + """ val = rospackexec(['plugins', '--attrib=plugin', pkg]) if val: return [tuple(x.split(' ')) for x in val.split('\n')] else: return [] - -## @return str: result of executing command (via subprocess). string will be strip()ed. def rosstackexec(args): + """ + @return: result of executing rosstack command (via subprocess). string will be strip()ed. + @rtype: str + @raise roslib.exceptions.ROSLibException: if rosstack command fails + """ val = (subprocess.Popen(['rosstack'] + args, stdout=subprocess.PIPE).communicate()[0] or '').strip() if val.startswith('rosstack:'): #rospack error message raise Exception(val) return val -## @return list: A list of the names of the packages which depend directly on pkg +def rosstack_depends_on(s): + """ + @param s: stack name + @type s: str + @return: A list of the names of the stacks which depend on s + @rtype: list + """ + return rosstackexec(['depends-on', s]).split() + def rosstack_depends_on_1(s): + """ + @param s: stack name + @type s: str + @return: A list of the names of the stacks which depend directly on s + @rtype: list + """ return rosstackexec(['depends-on1', s]).split() +def rosstack_depends(s): + """ + @param s: stack name + @type s: str + @return: A list of the names of the stacks which s depends on + @rtype: list + """ + return rosstackexec(['depends', s]).split() + +def rosstack_depends_1(s): + """ + @param s: stack name + @type s: str + @return: A list of the names of the stacks which s depends on directly + @rtype: list + """ + print "rossstack", s + return rosstackexec(['depends1', s]).split() + ## @return ServerProxy XML-RPC proxy to ROS master def get_master(): import xmlrpclib diff --git a/core/roslib/src/roslib/stack_manifest.py b/core/roslib/src/roslib/stack_manifest.py index d5052526..cad5153c 100644 --- a/core/roslib/src/roslib/stack_manifest.py +++ b/core/roslib/src/roslib/stack_manifest.py @@ -58,11 +58,10 @@ class StackManifest(roslib.manifestlib._Manifest): super(StackManifest, self).__init__('stack') ## @param stack_dir str: path to stack directory -## @param environ dict: environment dictionary ## @param required bool: require that the directory exist ## @return str: path to manifest file of stack ## @throws InvalidROSPkgException if required is True and manifest file cannot be located -def _stack_file_by_dir(stack_dir, required=True, environ=os.environ): +def _stack_file_by_dir(stack_dir, required=True): try: p = os.path.join(stack_dir, STACK_FILE) if not required and not os.path.exists(p): @@ -77,16 +76,12 @@ Stack '%(stack_dir)s' is improperly configured: no manifest file is present. raise ## @param stack str: stack name -## @param environ dict: environment dictionary ## @param required bool: require that the directory exist ## @return str: path to manifest file of stack ## @throws InvalidROSPkgException if required is True and manifest file cannot be located -def stack_file(stack, required=True, environ=os.environ): - # ros_root needs to be determined from the environment or else - # everything breaks when trying to launch nodes via ssh where the - # path isn't setup correctly. - d = roslib.stacks.get_stack_dir(stack, required, ros_root=environ[roslib.rosenv.ROS_ROOT]) - return _stack_file_by_dir(d, required, environ) +def stack_file(stack, required=True): + d = roslib.stacks.get_stack_dir(stack) + return _stack_file_by_dir(d, required) ## @internal ## Parse stack.xml file diff --git a/core/roslib/src/roslib/stacks.py b/core/roslib/src/roslib/stacks.py index 2f510cbe..c263df96 100644 --- a/core/roslib/src/roslib/stacks.py +++ b/core/roslib/src/roslib/stacks.py @@ -134,6 +134,10 @@ def list_stacks(env=os.environ): pkg_dirs = roslib.packages.get_package_paths(environ=env) stacks = [] + # ros is assumed to be at ROS_ROOT + if os.path.exists(os.path.join(ros_root, 'stack.xml')): + stacks.append('ros') + _dir_cache['ros'] = ros_root for pkg_root in pkg_dirs: for dir, dirs, files in os.walk(pkg_root, topdown=True): diff --git a/tools/roswtf/src/roswtf/__init__.py b/tools/roswtf/src/roswtf/__init__.py index a943ac6c..dc3301b4 100644 --- a/tools/roswtf/src/roswtf/__init__.py +++ b/tools/roswtf/src/roswtf/__init__.py @@ -118,9 +118,9 @@ def _roswtf_main(): from roswtf.context import WtfContext from roswtf.environment import wtf_check_environment, invalid_url from roswtf.graph import wtf_check_graph - from roswtf.packages import wtf_check_packages - from roswtf.roslaunchwtf import wtf_check_roslaunch_static, wtf_check_roslaunch_online - + import roswtf.packages + import roswtf.roslaunchwtf + import roswtf.stacks import roswtf.plugins static_plugins, online_plugins = roswtf.plugins.load_plugins() @@ -136,14 +136,19 @@ def _roswtf_main(): print "Package:",curr_package ctx = WtfContext.from_package(curr_package) #TODO: load all .launch files in package + elif os.path.isfile('stack.xml'): + curr_stack = os.path.basename(os.path.abspath('.')) + print "Stack:",curr_stack + ctx = WtfContext.from_stack(curr_stack) else: print "No package in context" ctx = WtfContext.from_env() # static checks wtf_check_environment(ctx) - wtf_check_packages(ctx) - wtf_check_roslaunch_static(ctx) + roswtf.packages.wtf_check(ctx) + roswtf.stacks.wtf_check(ctx) + roswtf.roslaunchwtf.wtf_check_static(ctx) for p in static_plugins: p(ctx) @@ -185,7 +190,7 @@ def _roswtf_main(): rospy.init_node('roswtf', anonymous=True) online_checks = True - wtf_check_roslaunch_online(ctx) + roswtf.roslaunchwtf.wtf_check_online(ctx) for p in online_plugins: online_checks = True diff --git a/tools/roswtf/src/roswtf/context.py b/tools/roswtf/src/roswtf/context.py index 6aecaa7f..ef413878 100644 --- a/tools/roswtf/src/roswtf/context.py +++ b/tools/roswtf/src/roswtf/context.py @@ -37,6 +37,7 @@ import sys import roslib.manifest import roslib.packages +import roslib.stacks import roslib.rosenv import roslib.scriptutil import roslib.substitution_args @@ -51,6 +52,7 @@ class WtfException(Exception): pass ## system we are attempting to debug class WtfContext(object): __slots__ = ['pkg', 'pkg_dir', 'pkgs', + 'stack', 'stack_dir', 'stacks', 'manifest_file', 'manifest', 'env', 'ros_root', 'ros_package_path', 'pythonpath', 'ros_master_uri', @@ -74,8 +76,13 @@ class WtfContext(object): # main package we are running self.pkg = None self.pkg_dir = None + # main stack we are running + self.stack = None + # - list of all packages involved in this check self.pkgs = [] + # - list of all stacks involved in this check + self.stacks = [] # manifest location of package that we are running self.manifest_file = None @@ -126,16 +133,29 @@ class WtfContext(object): ctx = WtfContext() ctx.launch_files = roslaunch_files _load_roslaunch(ctx, roslaunch_files) - # ctx.pkg initialized by _load_roslaunch + # ctx.pkg and ctx.stack initialized by _load_roslaunch _load_pkg(ctx, ctx.pkg) + _load_stack(ctx, ctx.stack) _load_env(ctx, env) return ctx - + + @staticmethod + ## @throws WtfException: if context state cannot be initialized + def from_stack(stack, env=os.environ): + ctx = WtfContext() + _load_stack(ctx, stack) + ctx.pkgs = roslib.stacks.packages_of(stack) + _load_env(ctx, env) + return ctx + @staticmethod ## @throws WtfException: if context state cannot be initialized def from_package(pkg, env=os.environ): ctx = WtfContext() _load_pkg(ctx, pkg) + stack = roslib.stacks.stack_of(pkg) + if stack: + _load_stack(ctx, stack) _load_env(ctx, env) return ctx @@ -165,11 +185,20 @@ def _load_pkg(ctx, pkg): ctx.pkgs = [pkg] + roslib.scriptutil.rospack_depends(pkg) try: ctx.pkg_dir = roslib.packages.get_pkg_dir(pkg) - ctx.manifest_file = roslib.manifest.manifest_file(pkg) - ctx.manifest_file = roslib.manifest.parse_file(ctx.manifest_file) except roslib.packages.InvalidROSPkgException: raise WtfException("Cannot locate manifest file for package [%s]"%pkg) +## utility for initializing WtfContext state +## @throws WtfException: if context state cannot be initialized +def _load_stack(ctx, stack): + ctx.stack = stack + ctx.stacks = [stack] + roslib.scriptutil.rosstack_depends(stack) + try: + ctx.stack_dir = roslib.stacks.get_stack_dir(stack) + except roslib.stacks.InvalidROSStackException: + raise WtfException("Cannot locate manifest file for stack [%s]"%stack) + + ## utility for initializing WtfContext state ## @throws WtfException: if context state cannot be initialized def _load_env(ctx,env): diff --git a/tools/roswtf/src/roswtf/packages.py b/tools/roswtf/src/roswtf/packages.py index b9beb66f..5cab376d 100644 --- a/tools/roswtf/src/roswtf/packages.py +++ b/tools/roswtf/src/roswtf/packages.py @@ -225,7 +225,7 @@ def cmakelists_package_valid(ctx): missing.remove('rospack') return missing -packages_warnings = [ +warnings = [ # disabling as it is too common and regular #(makefile_exists, # "The following packages have no Makefile:"), @@ -241,19 +241,19 @@ packages_warnings = [ (cmake_gensrv, 'The following packages need rosbuild_gensrv() in CMakeLists.txt:'), ] -packages_errors = [ +errors = [ (msgs_built, "Messages have not been built in the following package(s).\nYou can fix this by typing 'rosmake %(pkg)s':"), (srvs_built, "Services have not been built in the following package(s).\nYou can fix this by typing 'rosmake %(pkg)s':"), (manifest_rpath_flags, "The following packages have rpath issues in manifest.xml:"), ] -def wtf_check_packages(ctx): +def wtf_check(ctx): # no package in context to verify if not ctx.pkg: return - for r in packages_warnings: + for r in warnings: warning_rule(r, r[0](ctx), ctx) - for r in packages_errors: + for r in errors: error_rule(r, r[0](ctx), ctx) diff --git a/tools/roswtf/src/roswtf/roslaunchwtf.py b/tools/roswtf/src/roswtf/roslaunchwtf.py index f8221432..de02cc2a 100644 --- a/tools/roswtf/src/roswtf/roslaunchwtf.py +++ b/tools/roswtf/src/roswtf/roslaunchwtf.py @@ -295,7 +295,7 @@ static_roslaunch_errors = [ (roslaunch_load_check, "roslaunch load failed"), ] -def wtf_check_roslaunch_static(ctx): +def wtf_check_static(ctx): if not ctx.launch_files: return @@ -312,7 +312,7 @@ def wtf_check_roslaunch_static(ctx): def _load_online_ctx(ctx): ctx.roslaunch_uris = roslaunch.netapi.get_roslaunch_uris() -def wtf_check_roslaunch_online(ctx): +def wtf_check_online(ctx): _load_online_ctx(ctx) for r in online_roslaunch_warnings: warning_rule(r, r[0](ctx), ctx) diff --git a/tools/roswtf/src/roswtf/stacks.py b/tools/roswtf/src/roswtf/stacks.py new file mode 100644 index 00000000..7e0055b7 --- /dev/null +++ b/tools/roswtf/src/roswtf/stacks.py @@ -0,0 +1,93 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2009, Willow Garage, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Willow Garage, Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# Revision $Id: packages.py 6258 2009-09-22 22:08:20Z kwc $ + +import os +import time + +from roswtf.environment import paths, is_executable +from roswtf.rules import warning_rule, error_rule + +import roslib.msgs +import roslib.packages +import roslib.scriptutil +import roslib.stacks +import roslib.stack_manifest + +_packages_of_cache = {} +def _packages_of(d): + if d in _packages_of_cache: + return _packages_of_cache[d] + else: + _packages_of_cache[d] = pkgs = roslib.stacks.packages_of(d) + return pkgs + +def manifest_depends(ctx): + # This rule should probably be cache optimized + errors = [] + stack_list = roslib.stacks.list_stacks() + #print stack_list + for s in ctx.stacks: + s_deps = [] + s_pkgs = _packages_of(s) + for p in s_pkgs: + s_deps.extend(roslib.scriptutil.rospack_depends_1(p)) + pkg_dir = roslib.stacks.get_stack_dir(s) + m_file = roslib.stack_manifest.stack_file(s, True) + m = roslib.stack_manifest.parse_file(m_file) + for d in m.depends: + if not d.stack in stack_list: + errors.append("%s (%s does not exist)"%(m_file, d)) + else: + pkgs = _packages_of(d.stack) + if not [p for p in pkgs if p in s_deps]: + errors.append("%s (%s appears to be an unnecessary depend)"%(m_file, d)) + return errors + +warnings = [ + (manifest_depends, "The following stack.xml file list invalid dependencies:"), + + ] +errors = [ + ] + +def wtf_check(ctx): + # no package in context to verify + if not ctx.stacks: + return + + for r in warnings: + warning_rule(r, r[0](ctx), ctx) + for r in errors: + error_rule(r, r[0](ctx), ctx) +