#!/bin/bash
#
# Copyright (c) 2013 Tobias Weigl, tobias.weigl@bmw-carit.de
#
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
#
# Description
#
# This script provides various functionalities to check and protocol the current
# development state of the meta-ROS layer for Yocto.
#
#
# Global configuration
#
# The script version is printed in the report files for comparability reasons.
# The script version is a shortened representation of the internal version number.
# If an update is not relevant to the user, the internal version number should be
# increased in the latest sequence, which is not visible in the report files.
internal_version_number='v1.00.006'
script_version=$(echo $internal_version_number | sed 's/.[^.]*$//')
app_name='meta-ROS integration test and report assistant'
authors='Tobias Weigl (tobias.weigl@bmw-carit.de)'
#
# Path information the script uses ('./' for the Yocto project root)
path_to_script_relative_to_project_root='./meta-ros/scripts/'
path_to_topmost_recipes_directory_relative_to_project_root='./meta-ros/recipes-ros/'
path_to_project_root_relative_to_script='./../../'
report_folder_name='../test-reports'
#
# Autodetect
current_script_location="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
#
# Autohandle prerequisites
if [[ ! -d "$current_script_location/$report_folder_name" ]]; then
mkdir "$current_script_location/$report_folder_name"
fi
#
# Internal vars
txt_file_meta_information_indicator='#'
txt_file_comment_indicator='-'
txt_file_pkg_checking_kw='Checking'
txt_file_pkg_ok_kw='ok'
txt_file_meta_information_kw='Meta information'
#
list_all_available_recipes () {
find "$path_to_project_root_relative_to_script/$path_to_topmost_recipes_directory_relative_to_project_root/" -name '*.bb'
}
list_all_available_recipes_with_bakeable_name () {
list_all_available_recipes | sed 's/^.*\///' | sed 's/.bb$//' | sed 's/_.*//'
}
count_available_recipes () {
list_all_available_recipes | wc -l
}
update_timestamps () {
# By method call, we work on this timestamp only and just re-format it so we satisfy the constraint that all offered timestamps are based on the exact same time.
current_timestamp=$(date)
# Satisfies the constraints: Multiple files can be chronologically ordered by lexical sorting, Windows character set compatible.
human_readable_timestamp_filename_compatible=$(date --date="$current_timestamp" +%Y'.'%m'.'%d'-'%H'_'%M'_'%S)
# Satisfies the constraints: Easy and fast to read, no seconds.
timestamp_pretty_print=$(date --date="$current_timestamp" +%d'.'%m'.'%Y' '%H':'%M)
}
clear_output_file () {
output_file=$1
# It is error prone to use a combination of > and >> for file outputs, so we explicitly erase files by calling this method and use only >> afterwards.
echo -n "" > "$output_file"
}
bake_all () {
arg=$1
array=(`list_all_available_recipes_with_bakeable_name`)
array_length=${#array[*]}
echo -e "\nChecking $array_length recipes."
# Init the build environment
cd "$path_to_project_root_relative_to_script"
build_directory=build
source ./oe-init-env $build_directory
# Go to the meta-ros git repository location to gather information about commits.
cd ../meta-ros
current_git_hash=$(git rev-parse HEAD)
# The used git command returns an ISO_8601 timestamp. We crop the UTC-offset that can be a pos/neg(hh:mm) value, or Z.
# Satisfies the constraints: Multiple files can be chronologically ordered by lexical sorting, Windows character set compatible.
git_date_filename_compatible=$(git show -s --format="%ci" $current_git_hash | sed 's/\ [+ -].*$//' | sed 's/\ Z$//' | sed 's/-/\./g' | sed 's/:/_/g' | sed 's/\ /-/')
# Satisfies the constraints: Easy and fast to read, no seconds.
git_date_pretty_print_temp=$(git show -s --format="%ci" $current_git_hash | sed 's/:[0-9][0-9][^:].*$//' | sed 's/-/\./g')
# The date format comes as year-month-day, but we want day-month-year.
git_date_year=$(echo $git_date_pretty_print_temp | sed 's/\..*$//')
git_date_month=$(echo $git_date_pretty_print_temp | sed 's/^[^.]*\.//' | sed 's/\..*$//')
git_date_day=$(echo $git_date_pretty_print_temp | sed 's/^[^.]*\.[^.]*\.//' | sed 's/\ .*$//')
git_date_pretty_print="$git_date_day"."$git_date_month"."$git_date_year"' '$(echo $git_date_pretty_print_temp | sed 's/^.*\ //')
cd ../build
update_timestamps
reportname="report-$git_date_filename_compatible"
report_path_and_name=$current_script_location/$report_folder_name/$reportname
report_file_raw=$report_path_and_name'.raw'
report_file_txt=$report_path_and_name'.txt'
clear_output_file "$report_file_raw"
clear_output_file "$report_file_txt"
i=0
while [ $i -lt $array_length ]; do
activity="$txt_file_pkg_checking_kw $((i+1))/$array_length: ${array[$i]}"
echo $activity
if [ "$arg" = "txt" ]; then
# Try to bake every recipe. Store results in a report file. Just give $txt_file_pkg_ok_kw for working recipes, print more information on errors.
echo $activity | tee -a "$report_file_raw" >> "$report_file_txt"
bitbake ${array[$i]} | tee -a "$report_file_raw" | grep -i "^NOTE: Tasks Summary\|^Summary:\|^ERROR:\|^WARNING" | sed "s/NOTE.*succeeded\./$txt_file_pkg_ok_kw/" >> "$report_file_txt"
else
# Try to bake every recipe. Just give console output.
bitbake ${array[$i]}
fi
let i++
done
# If there was a text-file generated, compute now also meta information and insert it as header.
if [ "$arg" = "txt" ]; then
compute_meta_info_for_txt_file "$report_file_txt" "$current_git_hash" "$git_date_pretty_print" "$timestamp_pretty_print"
fi
cd ../"$path_to_script_relative_to_project_root"
cd "$report_folder_name"
ln -s -f "$reportname"'.html' ./'Latest_report'
cd ..
}
compute_meta_info_for_txt_file () {
report_file_txt=$1
current_git_hash=$2
git_date_pretty_print=$3
timestamp_pretty_print=$4
first_line=$(head -n 1 "$report_file_txt")
# Check if the meta information was already computed and the file format is valid.
if [[ "$first_line" != *"$txt_file_pkg_checking_kw"* ]]; then
echo "Error computing meta informations for the report."
return
fi
temporary_meta_header='tmp.txt'
temporary_result_file='tmp_result.txt'
number_of_recipes=$(echo $(head -n 1 "$report_file_txt") | sed 's/^.*\///' | sed 's/:.*$//')
number_of_succeeded_recipes=0
while read line; do
if [[ "$line" == "$txt_file_pkg_ok_kw" ]]; then
let number_of_succeeded_recipes++
fi
done < "$report_file_txt"
recipes_succeded_in_percent=$(($(($number_of_succeeded_recipes*100))/$number_of_recipes))
clear_output_file "$temporary_meta_header"
# To allow for greedy parsing later on, these lines have to be extended or modified carefully.
echo "$txt_file_comment_indicator""$txt_file_comment_indicator""$txt_file_comment_indicator" >> "$temporary_meta_header"
echo "$txt_file_comment_indicator""$txt_file_meta_information_kw" >> "$temporary_meta_header"
echo "$txt_file_meta_information_indicator""Report generation date: $timestamp_pretty_print" >> "$temporary_meta_header"
echo "$txt_file_meta_information_indicator""Recipes succeeded: $number_of_succeeded_recipes of $number_of_recipes, equaling ""$recipes_succeded_in_percent""%" >> "$temporary_meta_header"
echo "$txt_file_meta_information_indicator""Git hash: $current_git_hash" >> "$temporary_meta_header"
echo "$txt_file_meta_information_indicator""Git commit date: $git_date_pretty_print" >> "$temporary_meta_header"
echo "$txt_file_meta_information_indicator""General Info: This report was generated using the $app_name $script_version" >> "$temporary_meta_header"
echo "$txt_file_comment_indicator""$txt_file_comment_indicator""$txt_file_comment_indicator" >> "$temporary_meta_header"
cat "$temporary_meta_header" "$report_file_txt" > "$temporary_result_file"
rm "$report_file_txt"
rm "$temporary_meta_header"
mv "$temporary_result_file" "$report_file_txt"
}
# Provides a greedy txt-parser that writes an xml-file.
txtdata2xml () {
report_file_txt=$1
if [ "$report_file_txt" == "" ]; then
echo "There was an error reading the necessary txt-file for creating the xml-file."
return
fi
report_file_xml=$(echo "$report_file_txt" | sed 's/txt$/xml/')
clear_output_file "$report_file_xml"
echo '' >> "$report_file_xml"
echo '' >> "$report_file_xml"
while read line; do
if [[ "$line" == *"$txt_file_pkg_checking_kw"* ]]; then
if [[ "$line" != *"$txt_file_pkg_checking_kw"\ 1/* ]]; then
echo " " >> "$report_file_xml"
fi
echo " " >> "$report_file_xml"
echo " "$(echo $line | sed 's/^.*:\ //')"" >> "$report_file_xml"
elif [[ "$line" == *"$txt_file_pkg_ok_kw"* ]]; then
echo " $line" >> "$report_file_xml"
elif [[ "$line" == "$txt_file_meta_information_indicator"* ]]; then
echo " ""$(echo $line | sed 's/^.//')""" >> "$report_file_xml"
elif [[ "$line" == "$txt_file_comment_indicator"* ]]; then
:
else
echo " $line" >> "$report_file_xml"
fi
done < "$report_file_txt"
echo " " >> "$report_file_xml"
echo "" >> "$report_file_xml"
}
# Provides a greedy xml-parser that writes an html report.
xml2html () {
report_file_xml=$1
if [ "$report_file_xml" == "" ]; then
echo "There was an error reading the necessary xml-file for creating the html-file."
return
fi
report_file_html=$(echo "$report_file_xml" | sed 's/xml$/html/')
clear_output_file "$report_file_html"
### start multi-line echo
echo '
meta-ROS - bitbake report
meta-ROS - bitbake report
Available meta information:
' >> "$report_file_html"
### end multi-line echo
while read line; do
if [[ "$line" == *\
* ]]; then
echo '' >> "$report_file_html"
elif [[ "$line" == *\
* ]]; then
clean_meta_info=$( echo $line | sed 's/.*//' | sed 's/<\/meta_info>//' )
if [[ "$clean_meta_info" == *General\ Info* ]]; then
echo ' '"$clean_meta_info"'
' >> "$report_file_html"
else
echo ' '"$clean_meta_info"'
' >> "$report_file_html"
fi
elif [[ "$line" == *\* ]]; then
echo ' ' >> "$report_file_html"
elif [[ "$line" == *\
* ]]; then
echo -n ' ' >> "$report_file_html"
echo -n "$line" | sed 's/
//' | sed 's/<\/package>//' >> "$report_file_html"
echo ' ' >> "$report_file_html"
echo ' ' >> "$report_file_html"
echo '
' >> "$report_file_html"
elif [[ "$line" == *\* ]]; then
no_tags_line=$(echo $line | sed 's/.*//' | sed 's/<\/result_line>.*$//')
if [[ "$no_tags_line" == "$txt_file_pkg_ok_kw"* ]]; then
echo -n ' - '"$no_tags_line" >> "$report_file_html"
elif [[ "$no_tags_line" == Summary*Error* ]]; then
echo -n '
- '"$no_tags_line" >> "$report_file_html"
elif [[ "$no_tags_line" == Summary*failed* ]]; then
echo -n '
- '"$no_tags_line" >> "$report_file_html"
elif [[ "$no_tags_line" == NOTE*failed* ]]; then
echo -n '
- '"$no_tags_line" >> "$report_file_html"
elif [[ "$no_tags_line" == Summary*Warning* ]]; then
echo -n '
- '"$no_tags_line" >> "$report_file_html"
elif [[ "$no_tags_line" == *WARNING* ]]; then
echo -n '
- '"$no_tags_line" >> "$report_file_html"
elif [[ "$no_tags_line" == *ERROR* ]]; then
echo -n '
- '"$no_tags_line" >> "$report_file_html"
else
echo -n '
- '"$no_tags_line" >> "$report_file_html"
fi
echo '
' >> "$report_file_html"
elif [[ "$line" == *\* ]]; then
echo '
' >> "$report_file_html"
elif [[ "$line" == *\* ]]; then
echo ' ' >> "$report_file_html"
echo ' ' >> "$report_file_html"
echo ' ' >> "$report_file_html"
echo ' ' >> "$report_file_html"
fi
done < "$report_file_xml"
### start multi-line echo
echo '
' >> "$report_file_html"
### end multi-line echo
}
convert_all_available_xml_files_to_html_files () {
array=(`find "$report_folder_name/" -name '*.xml'`)
array_length=${#array[*]}
if [[ $array_length -lt 1 ]]; then
echo 'Error creating html-files: There are no xml-files available for conversion!'
else
echo "Creating $array_length new html-files!"
i=0
while [ $i -lt $array_length ]; do
xml2html "${array[$i]}"
let i++
done
fi
}
convert_all_available_txt_files_to_xml_files () {
array=(`find "$report_folder_name/" -name '*.txt'`)
array_length=${#array[*]}
if [[ $array_length -lt 1 ]]; then
echo 'Error creating xml-files: There are no txt-files available for conversion!'
else
echo "Creating $array_length new xml-files!"
i=0
while [ $i -lt $array_length ]; do
txtdata2xml "${array[$i]}"
let i++
done
fi
}
# Provides a greedy txt-parser that writes an html meta-report.
convert_all_available_txt_files_to_html_meta_report () {
# It is important here that all reports are chronologically sorted so that the meta report is easily understandable.
# Since the date format was chosen adequately, this can be achieved with sort -r.
array=(`find "$report_folder_name/" -name '*.txt' | sort -r`)
array_length=${#array[*]}
if [[ $array_length -lt 1 ]]; then
echo 'Error creating meta-report: There are no reports available for interpretation!'
return
fi
echo "Creating new meta-report from $array_length txt-files!"
update_timestamps
reportname="meta-report-$human_readable_timestamp_filename_compatible"
report_path_and_name=$current_script_location/$report_folder_name/$reportname
meta_report_file_html=$report_path_and_name'.html'
clear_output_file "$meta_report_file_html"
### start multi-line echo
echo '
meta-ROS - bitbake meta report
meta-ROS - bitbake meta report
Available meta information:
Report generation date: '"$timestamp_pretty_print"'
General Info: This meta report was generated using the '"$app_name $script_version"'
' >> "$meta_report_file_html"
### end multi-line echo
# Check the number of recipes of the latest commit. It is the benchmark value for the relative_build_process_progress_percentage.
number_of_recipes_of_latest_commit_extract ${array[0]}
# Check all available reports.
i=0
reports_succeeded=0
while [ $i -lt $array_length ]; do
txtdata2meta_report "${array[$i]}" "$meta_report_file_html"
let i++
done
# If there were errors reading the meta informations of a text file, display an error message.
if [[ $reports_succeeded -lt $array_length ]]; then
echo "Succeeded for $reports_succeeded!"
fi
echo '
' >> "$meta_report_file_html"
echo '
' >> "$meta_report_file_html"
echo '
' >> "$meta_report_file_html"
echo '' >> "$meta_report_file_html"
echo '' >> "$meta_report_file_html"
cd "$report_folder_name"
ln -s -f "$reportname"'.html' ./'Latest_meta-report'
cd ..
}
number_of_recipes_of_latest_commit_extract () {
latest_commit_report_file=$1
while read line; do
if [[ "$line" == \#Recipes\ succeeded* ]];then
number_of_recipes_of_latest_commit=$(echo "$line" | sed 's/^.*of\ //' | sed 's/,.*$//')
fi
done < "$latest_commit_report_file"
}
txtdata2meta_report () {
sub_report_txt=$1
meta_report_file_html=$2
# Check if the text report has meta informations. If not, it is probably uncomplete or in progress and we reject it.
second_line=$(head -n 2 "$sub_report_txt")
if [[ "$second_line" != *"$txt_file_meta_information_kw"* ]];then
return
fi
let reports_succeeded++
while read line; do
if [[ "$line" == \#Recipes\ succeeded* ]];then
succeeded_recipes=$(echo "$line" | sed 's/^#Recipes succeeded:\ //' | sed 's/\ of.*$//')
recipes_checked=$(echo "$line" | sed 's/^.*of\ //' | sed 's/,.*$//')
# For graphical rendering reasons, the outer bar of the progress bar can be at most 98px, since it has a 2px right-border.
current_commit_build_process_progress_percentage=$(echo "$line" | sed 's/^.*equaling\ //'| sed 's/%.*$//')
let98_current_commit_build_process_progress_percentage=$current_commit_build_process_progress_percentage
if [[ $let98_current_commit_build_process_progress_percentage -gt 98 ]]; then
let98_current_commit_build_process_progress_percentage=98
fi
elif [[ "$line" == \#Git\ hash:\ * ]];then
githash=$(echo "$line" | sed 's/^#Git hash:\ //')
elif [[ "$line" == \#Git\ commit\ date:\ * ]];then
git_date_pretty_print=$(echo "$line" | sed 's/^#Git\ commit\ date:\ //')
fi
done < "$sub_report_txt"
# Semantic convention for the meta report: The relative progress of a commit is min(arg1, arg2) in
# arg1: (successfully built recipes/the latest commit's number of recipes) in percent,
# arg2: (successfully built recipes/the current commit's number of recipes) in percent.
relative_build_process_progress_percentage1=$(($(($succeeded_recipes*100))/$number_of_recipes_of_latest_commit))
relative_build_process_progress_percentage2=$current_commit_build_process_progress_percentage
if [[ relative_build_process_progress_percentage1 -gt relative_build_process_progress_percentage2 ]]; then
relative_build_process_progress_percentage=$relative_build_process_progress_percentage2
else
relative_build_process_progress_percentage=$relative_build_process_progress_percentage1
fi
echo '
Git commit date: '"$git_date_pretty_print"'
Recipes succeeded: '"$succeeded_recipes"' of '"$recipes_checked"', equaling '"$current_commit_build_process_progress_percentage"'% (rel.: '"$relative_build_process_progress_percentage"'%)
Git hash: '"$githash"'
' >> "$meta_report_file_html"
}
bake () {
bake_all
}
bake_txtfile () {
bake_all txt
}
bake_xmlfile () {
bake_all txt
txtdata2xml "$report_path_and_name"'.txt'
}
bake_htmlfile () {
bake_all txt
txtdata2xml "$report_path_and_name"'.txt'
xml2html "$report_path_and_name"'.xml'
}
bake_html_and_metafile () {
bake_htmlfile
convert_all_available_txt_files_to_html_meta_report
}
create_htmlmetafile () {
convert_all_available_txt_files_to_html_meta_report
}
eval_args () {
args=$1
for arg in $args ; do
case "$arg" in
--ls)
list_all_available_recipes ;;
--lb)
list_all_available_recipes_with_bakeable_name ;;
--bc)
count_available_recipes ;;
--b)
bake ;;
--btxt)
bake_txtfile ;;
--bxml)
bake_xmlfile ;;
--bhtml)
bake_htmlfile ;;
--bhtmlmeta)
bake_html_and_metafile ;;
--htmlmeta)
create_htmlmetafile ;;
--t2x)
convert_all_available_txt_files_to_xml_files ;;
--x2h)
convert_all_available_xml_files_to_html_files ;;
--help) eval_args -h ;;
-h)
show_help ;;
*)
echo "Unknown command." ;;
esac
done
}
show_help () {
clear
echo "Name:"
echo "Integration test and report assistant for the meta-ROS layer"
echo ""
echo "Description:"
echo "--b: Bake all recipes and give console output"
echo "--bc: Count available meta-ROS recipes"
echo "--bhtml: Bake all recipes and save results to time-stamped html-file"
echo "--bhtmlmeta: Execute bhtml, htmlmeta"
echo "--btxt: Bake all recipes and save results to time-stamped txt+raw-files"
echo "--bxml: Bake all recipes and save results to time-stamped xml-file"
echo "--help or -h: Show this help page"
echo "--htmlmeta: Generate time stamped meta-report over all available reports"
echo "--ls: List available meta-ROS recipes with full name"
echo "--lb: List available meta-ROS recipes with bakeable name"
echo "--t2x: Create xml-reports for all available txt-reports"
echo "--x2h: Create html-reports for all available xml-reports"
echo ""
echo "Run the script using a described option as command line argument"
echo "No argument executes standard action: "$standard_action
}
#main call
standard_action='--bhtmlmeta'
if [ $# -eq 0 ]; then
eval_args $standard_action
elif [ $# -eq 1 ]; then
eval_args $1
else
echo "Error: Too many arguments."
fi