1. MPI_Allgatherv, version 0.6.2

2. a little modification in MPI_Gazebo_launch.sh and mygzserver
3. reduce link::update in worldUpdateBegin
4. modify time computing method in world.cc
This commit is contained in:
zhangshuai 2019-09-23 09:29:34 +08:00
parent 8b1c77a2fa
commit 2f8e7d4997
8 changed files with 1374 additions and 36 deletions

View File

@ -35,6 +35,9 @@
#include <gazebo/math/Helpers.hh>
#include "gazebo/util/system.hh"
#include <iomanip> //Added by zhangshuai 2019.09.09
#include <sys/time.h> //Added by zhangshuai 2019.09.09
namespace gazebo
{
/// \ingroup gazebo_event
@ -381,8 +384,26 @@ namespace gazebo
this->myDataPtr->signaled = true;
for (auto iter: this->myDataPtr->connections)
{
//std::cout << iter.first << std::endl; //zhangshuai test 2019.09.10
//struct timespec ts;
//clock_gettime(CLOCK_REALTIME, &ts);
//long cur_time;
//long update_time;
//cur_time = (double)ts.tv_sec + (double)ts.tv_nsec / 1.e9;<< std::scientific
//cur_time = ts.tv_nsec;
if (iter.second->on)
(*iter.second->callback)(_p);
//clock_gettime(CLOCK_REALTIME, &ts);
//update_time = ts.tv_nsec - cur_time;
//std::cout << "===== time1 : " << cur_time << std::endl; //zhangshuai test 2019.09.10
//std::cout << "===== time2 : " << ts.tv_nsec << std::endl; //zhangshuai test 2019.09.10
//std::cout << "===== OnUpdate time : " << update_time << std::endl; //zhangshuai test 2019.09.10
}
}

View File

@ -246,8 +246,32 @@ void Link::Load(sdf::ElementPtr _sdf)
}
}
this->connections.push_back(event::Events::ConnectWorldUpdateBegin(
boost::bind(&Link::Update, this, _1)));
//this->connections.push_back(event::Events::ConnectWorldUpdateBegin(
// boost::bind(&Link::Update, this, _1)));
// Modified by zhangshuai 2019.09.10 ----Begin
if (this->GetWorld()->GetFlag() == 1)
{
// Judge if the model to load is simulated in this gazebo
for (unsigned int i = 0; i < this->GetWorld()->GetDistribution()->GetGazeboIDPtr(this->GetWorld()->GetGazeboLocalID())->GetModelCount(); i++)
{
if (this->GetModel()->GetName() == this->GetWorld()->GetDistribution()->GetGazeboIDPtr(this->GetWorld()->GetGazeboLocalID())->GetModelName(i))
{
// Original part
this->connections.push_back(event::Events::ConnectWorldUpdateBegin(
boost::bind(&Link::Update, this, _1)));
// Original part
}
}
}
else
{
// Original part
this->connections.push_back(event::Events::ConnectWorldUpdateBegin(
boost::bind(&Link::Update, this, _1)));
// Original part
}
// Modified by zhangshuai 2019.09.10 ----End
std::string topicName = "~/" + this->GetScopedName() + "/wrench";
boost::replace_all(topicName, "::", "/");

View File

@ -107,6 +107,21 @@ double before_mpiTime = 0;
double mpiTime = 0;
double after_mpiTime = 0;
double wholeUpdataTime = 0;
double before_worldUpdateBeginTime2 = 0;
double worldUpdateBeginTime2 = 0;
double modelUpdateFuncTime2 = 0;
double updateCollisionTime2 = 0;
double loggingTime2 = 0;
double before_updatePhysicsTime2 = 0;
double updatePhysicsTime2 = 0;
double before_mpiTime2 = 0;
double mpiTime2 = 0;
double after_mpiTime2 = 0;
double wholeUpdataTime2 = 0;
double simtime = 0;
double realtime = 0;
uint64_t start_iterations = 0;
#endif
// Added by zhangshuai 2019.04.03 for count time ----End
@ -271,7 +286,13 @@ void World::Load(sdf::ElementPtr _sdf)
this->dataPtr->lightFactoryPub = this->dataPtr->node->Advertise<msgs::Light>(
"~/factory/light");
// Added by zhangshuai based on zenglei 2019.04.23 ----Begin
// Added by zhangshuai based on zenglei 2019.09.23 ----Begin
std::cout << " ____________________________________________________________" << std::endl;
std::cout << "| This is Gazebo-Exercise-MPI 0.6 |" << std::endl;
std::cout << "| by AIRC-01 |" << std::endl;
std::cout << "|____________________________________________________________|" << std::endl;
// Added by zhangshuai based on zenglei 2019.04.23 ----End
if (this->dataPtr->sdf->HasElement("distribution"))
{
sdf::ElementPtr distributionElem = this->dataPtr->sdf->GetElement("distribution");
@ -1048,26 +1069,44 @@ void World::Update()
#ifdef USE_COUNT_TIME
// std::cout << "===== end iterations:\t" << this->dataPtr->iterations << std::endl;
if (this->dataPtr->iterations == 0)
{
before_worldUpdateBeginTime2 = before_worldUpdateBeginTime;
worldUpdateBeginTime2 = worldUpdateBeginTime;
modelUpdateFuncTime2 = modelUpdateFuncTime;
updateCollisionTime2 = updateCollisionTime;
loggingTime2 = loggingTime;
before_updatePhysicsTime2 = before_updatePhysicsTime;
updatePhysicsTime2 = updatePhysicsTime;
before_mpiTime2 = before_mpiTime;
mpiTime2 = mpiTime;
after_mpiTime2 = after_mpiTime;
wholeUpdataTime2 = wholeUpdataTime;
simtime = this->GetSimTime().Double();
realtime = this->GetRealTime().Double();
start_iterations = this->dataPtr->iterations;
}
if (this->dataPtr->iterations == 100000)
{
// if (this->gazeboLocalID == 0)
// {
std::cout << "===== average before_worldUpdateBeginTime:\t" << before_worldUpdateBeginTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average worldUpdateBeginTime:\t" << worldUpdateBeginTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average modelUpdateFuncTime:\t" << modelUpdateFuncTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average updateCollisionTime:\t" << updateCollisionTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average loggingTime:\t" << loggingTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average before_updatePhysicsTime:\t" << before_updatePhysicsTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average updatePhysicsTime:\t" << updatePhysicsTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average before_mpiTime:\t" << before_mpiTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average mpiTime:\t" << mpiTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average after_mpiTime:\t" << after_mpiTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
double wholeMpiTime = before_mpiTime + mpiTime + after_mpiTime;
std::cout << "===== average wholeMpiTime:\t" << wholeMpiTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
double wholeAddTime = before_worldUpdateBeginTime + worldUpdateBeginTime + modelUpdateFuncTime + updateCollisionTime + loggingTime + before_updatePhysicsTime + updatePhysicsTime + before_mpiTime + mpiTime + after_mpiTime;
std::cout << "===== average wholeAddTime:\t" << wholeAddTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average wholeUpdateTime:\t" << wholeUpdataTime * 1000.0 / this->dataPtr->iterations << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average RTF:\t" << this->GetSimTime().Double() / this->GetRealTime().Double() << "\t =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average before_worldUpdateBeginTime:\t" << (before_worldUpdateBeginTime - before_worldUpdateBeginTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average worldUpdateBeginTime:\t" << (worldUpdateBeginTime - worldUpdateBeginTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average modelUpdateFuncTime:\t" << (modelUpdateFuncTime - modelUpdateFuncTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average updateCollisionTime:\t" << (updateCollisionTime - updateCollisionTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average loggingTime:\t" << (loggingTime - loggingTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average before_updatePhysicsTime:\t" << (before_updatePhysicsTime - before_updatePhysicsTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average updatePhysicsTime:\t" << (updatePhysicsTime - updatePhysicsTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average before_mpiTime:\t" << (before_mpiTime - before_mpiTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average mpiTime:\t" << (mpiTime - mpiTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average after_mpiTime:\t" << (after_mpiTime - after_mpiTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
double wholeMpiTime = before_mpiTime + mpiTime + after_mpiTime - (before_mpiTime2 + mpiTime2 + after_mpiTime2);
std::cout << "===== average wholeMpiTime:\t" << wholeMpiTime * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
double wholeAddTime = before_worldUpdateBeginTime + worldUpdateBeginTime + modelUpdateFuncTime + updateCollisionTime + loggingTime + before_updatePhysicsTime + updatePhysicsTime + before_mpiTime + mpiTime + after_mpiTime - (before_worldUpdateBeginTime2 + worldUpdateBeginTime2 + modelUpdateFuncTime2 + updateCollisionTime2 + loggingTime2 + before_updatePhysicsTime2 + updatePhysicsTime2 + before_mpiTime2 + mpiTime2 + after_mpiTime2);
std::cout << "===== average wholeAddTime:\t" << wholeAddTime * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average wholeUpdateTime:\t" << (wholeUpdataTime - wholeUpdataTime2) * 1000.0 / (this->dataPtr->iterations - start_iterations) << "\tms =====\t" << this->gazeboLocalID << std::endl;
std::cout << "===== average RTF:\t" << (this->GetSimTime().Double() - simtime) / (this->GetRealTime().Double() - realtime) << "\t =====\t" << this->gazeboLocalID << std::endl;
// }
}
#endif

View File

@ -3,10 +3,12 @@
# Author: Zhang Shuai #
# Date: 2019-08-26 #
# Description: End a MPI Gazebo simulation task #
# Usage: ./MPI_Gazebo_end.sh [hostfile] #
# Usage: ./MPI_Gazebo_end.sh [hostfile] [shell's path] #
# Example: ./MPI_Gazebo_launch.sh hostfile ~/Git/Gazebo_Distributed_MPI/mpi_run #
#####################################################################################################################################
host_file=$1
shell_path=$2
host_count=0
hosts=()
@ -29,10 +31,10 @@ do
echo "host_name:${hosts[$i]}"
done
#登入各节点kill相关进程以结束仿真1.目前方式比较暴力,后续改进2.cd ~/Git/Gazebo_Distributed_MPI/mpi_run 还要进行灵活性优化
#登入各节点kill相关进程以结束仿真1.目前方式比较暴力,后续改进
for((i=0;i<host_count;i++))
do
ssh -f -n ${hosts[$i]} > /dev/null 2>&1 "cd ~/Git/Gazebo_Distributed_MPI/mpi_run;
ssh -f -n ${hosts[$i]} > /dev/null 2>&1 "cd $shell_path;
./kill.sh;
./kill.sh;
./kill.sh;

View File

@ -1,22 +1,26 @@
#!/bin/bash
#####################################################################################################################################
# Author: Zhang Shuai #
# Date: 2019-05-23 #
# Description: launch a MPI Gazebo simulation task #
# Usage: ./MPI_Gazebo_launch.sh [ros packdge's name] [launch file's name] [workspace's path] [world's path and name] [hostfile] #
#####################################################################################################################################
###################################################################################################################################################
# Author: Zhang Shuai #
# Date: 2019-05-23 #
# Description: launch a MPI Gazebo simulation task #
# Usage: ./MPI_Gazebo_launch.sh [ros packdge's name] [launch file's name] [workspace's path] [world's path and name] [hostfile] [shell's path] #
# Example: ./MPI_Gazebo_launch.sh rosplane_sim fixedwing_text_three_100_withoutfast_mpi.launch ~/zhangshuai/DA #
# ~/zhangshuai/DA/src/da/rosplane/rosplane_sim/worlds/flying_mpi_100.world hostfile ~/Git/Gazebo_Distributed_MPI/mpi_run #
###################################################################################################################################################
packdge_name=$1
launch_file_name=$2
workspace_path=$3
world_file=$4
host_file=$5
shell_path=$6
echo $packdge_name
echo $launch_file_name
echo $workspace_path
echo $world_file
echo $host_file
echo $shell_path
host_count=0
hosts=()
@ -39,26 +43,29 @@ do
echo "host_name:${hosts[$i]}"
done
#登入各节点启动roscore source还有改进空间ROS运行的工作目录应该作为参数传进来
#登入各节点启动roscore
for((i=0;i<host_count;i++))
do
ssh -f -n ${hosts[$i]} "source $workspace_path/devel/setup.bash;
export ROS_HOSTNAME=localhost;
roscore;
echo roscore_done!"
echo roscore_done!;
rosparam set /use_sim_time true;
rosparam get /use_sim_time"
echo "${hosts[$i]}:roscore_good"
done
sleep 5s
#登入第一个节点启动MPI的gzserver并成为各自节点上的ROS node
#登入第一个节点启动MPI的gzserver并成为各自节点上的ROS node ## btl_tcp_if_exclude / btl_tcp_if_include
ssh -f -n ${hosts[0]} "cd ~/Git/Gazebo_Distributed_MPI/mpi_run;
mpiexec --mca btl_tcp_if_exclude docker0 -hostfile $host_file -np 1 mygzserver $world_file $workspace_path : -np 1 mygzserver $world_file $workspace_path;
mpiexec --mca btl_tcp_if_include eno2d1 -hostfile $host_file -np 1 mygzserver $world_file $workspace_path : -np 1 mygzserver $world_file $workspace_path;
echo mpiexec_done!"
echo "${hosts[0]}:mpiexec_good"
sleep 10s
#登入各节点roslaunch加载模型source还有改进空间ROS运行的工作目录应该作为参数传进来
#登入各节点roslaunch加载模型
for((i=0;i<host_count;i++))
do
ssh -f -n ${hosts[$i]} "source $workspace_path/devel/setup.bash;
@ -70,7 +77,7 @@ do
echo "${hosts[$i]}:roslaunch_good"
done
sleep 20s
sleep 100s
#登入各节点开始仿真
for((i=0;i<host_count;i++))
@ -83,10 +90,10 @@ done
sleep 5m
#登入各节点kill相关进程以结束仿真1.目前方式比较暴力,后续改进2.cd ~/Git/Gazebo_Distributed_MPI/mpi_run 还要进行灵活性优化
#登入各节点kill相关进程以结束仿真1.目前方式比较暴力,后续改进
for((i=0;i<host_count;i++))
do
ssh -f -n ${hosts[$i]} > /dev/null 2>&1 "cd ~/Git/Gazebo_Distributed_MPI/mpi_run;
ssh -f -n ${hosts[$i]} > /dev/null 2>&1 "cd $shell_path;
./kill.sh;
./kill.sh;
./kill.sh;

View File

@ -5,5 +5,6 @@ source /opt/ros/kinetic/setup.bash
source $2/devel/setup.bash
export DISPLAY=:0.0
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib/gazebo-7/plugins:/opt/ros/kinetic/lib:/opt/ros/kinetic/lib/x86_64-linux-gnu
/usr/local/bin/gzserver -u -e ode --verbose $1 -s /opt/ros/kinetic/lib/libgazebo_ros_paths_plugin.so -s /opt/ros/kinetic/lib/libgazebo_ros_api_plugin.so

View File

@ -0,0 +1,128 @@
<!-- This is a launch file that runs the bare minimum requirements to get -->
<!-- gazebo running for a fixed-wing aircraft -->
<launch>
<group ns="bebop_0">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_0" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_0" />
<arg name="x" value="0.0" />
<arg name="y" value="0.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_1">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_1" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_1" />
<arg name="x" value="0.0" />
<arg name="y" value="5.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_2">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_2" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_2" />
<arg name="x" value="0.0" />
<arg name="y" value="10.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_3">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_3" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_3" />
<arg name="x" value="0.0" />
<arg name="y" value="15.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_4">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_4" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_4" />
<arg name="x" value="0.0" />
<arg name="y" value="20.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_5">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_5" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_5" />
<arg name="x" value="0.0" />
<arg name="y" value="25.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_6">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_6" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_6" />
<arg name="x" value="0.0" />
<arg name="y" value="30.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_7">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_7" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_7" />
<arg name="x" value="0.0" />
<arg name="y" value="35.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_8">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_8" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_8" />
<arg name="x" value="0.0" />
<arg name="y" value="40.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<group ns="bebop_9">
<include file="$(find hector_quadrotor_gazebo)/launch/spawn_quadrotor.launch">
<arg name="name" value="bebop_9" />
<arg name="model" value="$(find hector_quadrotor_description)/urdf/quadrotor_with_downward_cam.gazebo.xacro"/>
<arg name="tf_prefix" value="bebop_9" />
<arg name="x" value="0.0" />
<arg name="y" value="45.0" />
<arg name="z" value="0.186" />
</include>
<node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/>
</group>
<!-- <node name="simple_takeoff" pkg="hector" type="hector_simple_takeoff"/> -->
</launch>