3.1. Relevant Information for Bug ReportingΒΆ
In case you cannot investigate and resolve the issue by yourself using this document, make sure you open a ticket on your 6WIND Customer Zone with the relevant troubleshooting information.
This information can be generated and exported using the
troubleshooting-report.sh script.
Note
If you are in an OpenStack environment, call the script providing as arguments:
--controllerinside a controller node--computeinside a compute node--networkinside a network node
Also make sure credentials for access to nova are exported in the environment
(i.e.: OS_USERNAME, OS_PASSWORD, OS_PROJECT_DOMAIN_ID, etc.).
If possible, reproduce your issue with debug info enabled. Enable debug for the shortest time possible, as it produces a flabbergasting amount of log.
To enable debug, look at the OpenStack logs section.
We recommend installing the sos package before calling the following script,
to leverage the distribution bug reporting mechanism (sosreport). Once
installed, troubleshooting-report.sh will detect it and run it.
troubleshooting-report.sh is provided here for information:
#!/bin/sh
#
# Copyright 2016 6WIND S.A.
cleanup()
{
[ -d "$TMPDIR" ] && rm -rf $TMPDIR
}
usage_option()
{
printf "\t%s" "$1"
[ -n "$3" ] && printf " %s" "$3"
printf "\t\t%s\n" "$2"
}
usage()
{
printf "%s\n\n" "$0 [-h|--help] [-c|--compute] [-C|--controller] [-N|--network] [-f|--file <file path>] [-o|--no-core-file] [-O|--clean-core-files] [-e|--extra <file path>]"
usage_option -h "display this help"
usage_option -c "Use this for OpenStack compute nodes"
usage_option -C "Use this for OpenStack controller nodes"
usage_option -N "Use this for OpenStack network nodes"
usage_option -f "Output file path"
usage_option -o "Do not archive core files"
usage_option -O "Cleanup core files after generating the report"
usage_option -e "Add an external file in the report archive"
}
parse_args()
{
# Turn long options into short ones
for arg in "$@"; do
shift
case "$arg" in
"--help") set -- "$@" "-h" ;;
"--controller") set -- "$@" "-C" ;;
"--compute") set -- "$@" "-c" ;;
"--network") set -- "$@" "-N" ;;
"--file") set -- "$@" "-f" ;;
"--no-core-file")set -- "$@" "-o" ;;
"--clean-core-files")set -- "$@" "-O" ;;
"--extra") set -- "$@" "-e" ;;
*) set -- "$@" "$arg"
esac
done
OPTIND=1
while getopts hcCNf:oOe: name
do
case "$name" in
c) COMPUTE="yes" && OPENSTACK="yes" && \
SUFFIX="${SUFFIX}_compute" ;;
C) CONTROLLER="yes" && OPENSTACK="yes" && \
SUFFIX="${SUFFIX}_controller" ;;
N) NETWORK="yes" && OPENSTACK="yes" && \
SUFFIX="${SUFFIX}_network" ;;
f) ARCHIVE=${OPTARG} ;;
o) NO_CORE_FILE="yes" ;;
O) CLEAN_CORE_FILES="yes" ;;
e) EXTRAFILES="${OPTARG} ${EXTRAFILES}" ;;
h) usage && exit 0 ;;
*) usage && exit 1 ;;
esac
done
}
# Check whether the given command is executable, and if the timeout utility is
# available, run the command using a $TIMEOUT_S seconds timeout.
exec_cmd()
{
local netns_cmd="$(echo $@ | sed -n 's/^ip netns exec [^ ]\+ \([^ ]\+\).*$/\1/p')"
# $cmd contains the command to check for without arguments,
# stripped from any leading `ip netns exec XXX`.
cmd=${netns_cmd:-$1}
[ -x "$(command -v $cmd)" ] && $TIMEOUT $@
}
get_linux_info()
{
local ns= name=$1
if [ -n "$name" ]; then
ns="ip netns exec $name"
else
name=main
fi
# information on all known links
for iface in $($ns ip link show | sed -n 's/^[^:]\+: \([^:]\+\):.*$/\1/p'); do
$ns fp-cli dpdk-port-stats $iface > $BUGDIR/$name-fp-cli_dpdk-port-stats_$iface.txt 2>&1
$ns fp-cli dpdk-port-offload $iface > $BUGDIR/$name-fp-cli_dpdk-port-offload_$iface.txt 2>&1
$ns fp-cli dpdk-port-advertise $iface > $BUGDIR/$name-fp-cli_dpdk-port-advertise_$iface.txt 2>&1
done
# interfaces, addresses, routes, neighbours and IPsec
$ns ip -detail -statistics link > $BUGDIR/$name-ip_link.txt 2>$LOGDIR/$name-ip_link.log
$ns ip -detail address > $BUGDIR/$name-ip_address.txt 2>$LOGDIR/$name-ip_address.log
# Adding pbr, splitting routes by tables
$ns ip -detail rule > $BUGDIR/$name-ip_rule.txt 2>$LOGDIR/$name-ip_rule.log
for table in $($ns ip rule | sed 's/^[ \t]*//;s/[ \t]*$//' | rev | cut -d" " -f 1 | rev); do
if ! echo $table | grep -q "l3mdev"; then
$ns ip -detail route show table $table > $BUGDIR/$name-ip_route-table-$table.txt 2>&1
fi
done
$ns ip -6 -detail rule > $BUGDIR/$name-ip_rule6.txt 2>$LOGDIR/$name-ip_rule6.log
for table in $($ns ip -6 rule | sed 's/^[ \t]*//;s/[ \t]*$//' | rev | cut -d" " -f 1 | rev); do
if ! echo $table | grep -q "l3mdev"; then
$ns ip -6 -detail route show table $table > $BUGDIR/$name-ip_route6-table-$table.txt 2>&1
fi
done
$ns ip neigh > $BUGDIR/$name-ip_neigh.txt 2>$LOGDIR/$name-ip_neigh.log
$ns ip -6 neigh > $BUGDIR/$name-ip_neigh6.txt 2>$LOGDIR/$name-ip_neigh6.log
# Adding l3vrf linux side
$ns ip vrf > $BUGDIR/$name-ip-l3vrf.txt 2>&1
if ! grep -q "No command" "$BUGDIR/$name-ip-l3vrf.txt" && ! grep -q "No VRF has been configured" "$BUGDIR/$name-ip-l3vrf.txt"; then
$ns ip vrf | tail -n +3 | while read -r line; do
vrf=`echo $line | cut -d" " -f 1`
$ns ip -detail route show vrf $vrf > $BUGDIR/$name-ip-route-l3vrf-$vrf.txt 2>&1
$ns ip -6 -detail route show vrf $vrf > $BUGDIR/$name-ip-route6-l3vrf-$vrf.txt 2>&1
done
fi
$ns ip -statistics xfrm policy > $BUGDIR/$name-ip_xfrm_policy.txt 2>$LOGDIR/$name-ip_xfrm_policy.log
$ns ip -statistics xfrm state > $BUGDIR/$name-ip_xfrm_state.txt 2>$LOGDIR/$name-ip_xfrm_state.log
# get fpn0 stats
if [ "$name" = "vrf0" ]; then
fpexec ethtool -S fpn0 > $BUGDIR/$name-ethtool_S_fpn0.txt 2>&1
fi
# active network connections
$ns netstat -anp > $BUGDIR/$name-nestat_anp.txt 2>$LOGDIR/$name-nestat_anp.log
# Netfilter
exec_cmd $ns iptables-save -c > $BUGDIR/$name-iptables_save.txt 2>$LOGDIR/$name-iptables_save.log
exec_cmd $ns ip6tables-save -c > $BUGDIR/$name-ip6tables_save.txt 2>$LOGDIR/$name-ip6tables_save.log
exec_cmd $ns ebtables-save > $BUGDIR/$name-ebtables_save.txt 2>$LOGDIR/$name-ebtables_save.log
exec_cmd $ns ipset save > $BUGDIR/$name-ipset_save.txt 2>$LOGDIR/$name-ipset_save.log
exec_cmd $ns ipset list > $BUGDIR/$name-ipset_list.txt 2>$LOGDIR/$name-ipset_list.log
# Conntrack
exec_cmd $ns conntrack -L > $BUGDIR/$name-conntrack.txt 2>&1
# Check bridge info
exec_cmd $ns brctl show > $BUGDIR/$name-brctl_show.txt 2>$LOGDIR/$name-brctl_show.log
exec_cmd $ns ovs-vsctl show > $BUGDIR/$name-ovs_vsctl_show.txt 2>&1
}
get_system_info()
{
# system topology
exec_cmd lstopo-no-graphics --output-format xml > $BUGDIR/lstopo.xml 2>$LOGDIR/lstopo.log
# processors hierarchy
cp /proc/cpuinfo $BUGDIR/cpuinfo.txt
exec_cmd lscpu > $BUGDIR/lscpu.txt 2>$LOGDIR/lscpu.log
# interrupts
cp /proc/interrupts $BUGDIR/interrupts.txt
# memory record
cp /proc/meminfo $BUGDIR/meminfo.txt
exec_cmd vmstat -ws > $BUGDIR/vmstat_ws.txt 2>$LOGDIR/vmstat_ws.log
exec_cmd numastat -zm > $BUGDIR/numastat_zm.txt 2>$LOGDIR/numastat_zm.log
exec_cmd numastat -zs > $BUGDIR/numastat_zs.txt 2>$LOGDIR/numastat_zs.log
# PCI peripherals
exec_cmd lspci -vvv > $BUGDIR/lspci.txt 2>$LOGDIR/lspci.log
# DMI/SMBIOS
exec_cmd dmidecode > $BUGDIR/dmidecode.txt 2>$LOGDIR/dmidecode.log
# kernel version, logs, cmdline and loaded modules
uname -a > $BUGDIR/uname.txt 2>$LOGDIR/uname.log
dmesg > $BUGDIR/dmesg.txt 2>$LOGDIR/dmesg.log
cp /proc/cmdline $BUGDIR/cmdline.txt
lsmod > $BUGDIR/lsmod.txt 2>$LOGDIR/lsmod.log
# distribution
exec_cmd lsb_release -a > $BUGDIR/lsb_release.txt 2>&1
exec_cmd cp /etc/os-release $BUGDIR/os_release.txt 2>&1
# services list
exec_cmd service --status-all > $BUGDIR/service_status_all.txt 2>&1
exec_cmd systemctl list-units > $BUGDIR/systemctl_list_units.txt 2>&1
# logs
exec_cmd journalctl --all --no-pager > $BUGDIR/journal.txt 2>$LOGDIR/journal.log
exec_cmd journalctl --all --no-pager --priority warning..warning \
> $BUGDIR/journalctl-warnings.txt 2>$LOGDIR/journalctl-warnings.log
exec_cmd journalctl --all --no-pager --priority err \
> $BUGDIR/journalctl-errors.txt 2>$LOGDIR/journalctl-errors.log
exec_cmd journalctl --since yesterday > $BUGDIR/journal-last-24h.txt \
2>$LOGDIR/journal-last-24h.log
[ -f "/var/log/syslog" ] && cp /var/log/syslog $BUGDIR/syslog.txt
[ -d "/var/log/libvirt" ] && cp -r /var/log/libvirt $BUGDIR/libvirt
[ -f "/var/log/netlimits.log" ] && cp /var/log/netlimits.log \
$BUGDIR/6WIND-vRouter-license-usage-report.txt
[ -f "/var/log/netlimits.log.sig" ] && cp /var/log/netlimits.log.sig \
$BUGDIR/6WIND-vRouter-license-usage-report.txt.sig
# processes list
ps auxww > $BUGDIR/ps_auxww.txt 2>$LOGDIR/ps_auxww.log
# cpuset
[ -d "/dev/cpuset" ] && cp -r /dev/cpuset $BUGDIR/cpuset 2>/dev/null
# /dev
ls -al /dev > $BUGDIR/ls_al_dev.txt 2>$LOGDIR/ls_al_dev.log
# IRQ affinity
find /proc/irq -maxdepth 1 -mindepth 1 -print -type d -exec \
cat '{}/smp_affinity' \; > $BUGDIR/proc_irq_smp_affinity.txt \
2>$LOGDIR/proc_irq_smp_affinity.log
# mounted partitions
exec_cmd mount > $BUGDIR/mount.txt 2>$LOGDIR/mount.log
exec_cmd vrfctl list > $BUGDIR/vrf_list.txt 2>&1
# shared memory usage
exec_cmd du -c -h /dev/shm/* | sort -rh > $BUGDIR/shm_usage.txt 2>$LOGDIR/shm_usage.log
}
get_coredump()
{
if [ "$NO_CORE_FILE" != 'yes' ] && ls /var/lib/systemd/coredump/core* >/dev/null 2>&1
then
mkdir $BUGDIR/coredump
cp /var/lib/systemd/coredump/core* $BUGDIR/coredump/
coredumpctl info > $BUGDIR/coredump/coredumpctl_info.txt 2>$LOGDIR/coredump/coredumpctl_info.log
if [ "$CLEAN_CORE_FILES" = 'yes' ]; then
rm -f /var/lib/systemd/coredump/core*
fi
fi
}
get_global_fp_info()
{
if [ -x "$(command -v fast-path.sh)" ]; then
# Record your fp configuration:
exec_cmd fp-conf-tool -DSFv > $BUGDIR/fp_config.txt \
2>$LOGDIR/fp_config.log
# Get machine hardware info summary
exec_cmd fp-conf-tool -m > $BUGDIR/hardware_info.txt \
2>$LOGDIR/hardware_info.log
# Copy env files
mkdir -p $BUGDIR/etc_6wind
[ -f "/etc/fast-path.env" ] && cp /etc/fast-path.env $BUGDIR/etc_6wind
[ -f "/etc/cpuset.env" ] && cp /etc/cpuset.env $BUGDIR/etc_6wind
[ -f "/etc/fp-vdev.ini" ] && cp /etc/fp-vdev.ini $BUGDIR/etc_6wind
[ -f "/etc/cmgr.env" ] && cp /etc/cmgr.env $BUGDIR/etc_6wind
[ -f "/etc/fp-daemons.env" ] && cp /etc/fp-daemons.env $BUGDIR/etc_6wind
[ -f "/etc/fps-fp.env" ] && cp /etc/fps-fp.env $BUGDIR/etc_6wind
[ -f "/etc/hitflags.env" ] && cp /etc/hitflags.env $BUGDIR/etc_6wind
[ -f "/etc/linux-fp-sync.env" ] && cp /etc/linux-fp-sync.env $BUGDIR/etc_6wind
[ -f "/etc/vrfd.env" ] && cp /etc/vrfd.env $BUGDIR/etc_6wind
[ -f "/etc/vrf.env" ] && cp /etc/vrf.env $BUGDIR/etc_6wind
# Record what 6windgate version is used
if [ -x "$(command -v dpkg)" ]; then
exec_cmd dpkg -s 6windgate-fp > $BUGDIR/fp_version.txt 2>$LOGDIR/fp_version.log
else
if exec_cmd yum list installed 6windgate-fp 2>/dev/null >/dev/null; then
exec_cmd yum info 6windgate-fp > $BUGDIR/fp_version.txt 2>$LOGDIR/fp_version.log
else
exec_cmd yum info virtual-accelerator-fp > $BUGDIR/fp_version.txt 2>$LOGDIR/fp_version.log
fi
fi
# in buildroot, /etc/issue contains 6WG version
[ -f "/etc/issue" ] && cp /etc/issue $BUGDIR/etc_issue.txt
# Copy fp logs
[ -f "/var/log/fast-path.log" ] && \
cp /var/log/fast-path.log $BUGDIR/fast-path.log
[ -f "/var/log/messages" ] && \
cp /var/log/messages $BUGDIR/messages.log
exec_cmd linux-fp-sync.sh status > $BUGDIR/linux_fp_sync_status.txt \
2>$LOGDIR/linux_fp_sync_status.log
fast-path.sh status > $BUGDIR/fast_path_status.txt \
2>$LOGDIR/fast_path_status.log
if grep -q "fp-.*not running" "$BUGDIR/fast_path_status.txt"; then
FP_RUNNING="false"
else
FP_RUNNING="true"
fi
fi
# Copy fp info
if [ "$FP_RUNNING" = "true" ]; then
# compilation options used for your FP:
$TIMEOUT fp-cli conf compiled > $BUGDIR/fp_compil_options.txt 2>&1
$TIMEOUT fp-cli stats percore > $BUGDIR/fp_cli_dump_stats.txt 2>$LOGDIR/fp_cli_dump_stats.log
$TIMEOUT fp-cli fp-state > $BUGDIR/fp_cli_fp_state.txt 2>$LOGDIR/fp_cli_fp_state.log
$TIMEOUT fp-cli filling > $BUGDIR/fp_cli_filling.txt 2>$LOGDIR/fp_cli_filling.log
if [ -n "$(fp-cli conf compiled | grep CONFIG_MCORE_ARCH_DPDK=y)" ]; then
$TIMEOUT fp-cli dpdk-debug-pool | grep common_pool_count > $BUGDIR/fp_cli_dpdk_pool.txt 2>$LOGDIR/fp_cli_dpdk_pool.log
fi
$TIMEOUT fp-cli iface > $BUGDIR/fp_cli_iface.txt 2>$LOGDIR/fp_cli_iface.log
$TIMEOUT fp-cli neigh4 > $BUGDIR/fp_cli_neigh4.txt 2>$LOGDIR/fp_cli_neigh4.log
$TIMEOUT fp-cli neigh6 > $BUGDIR/fp_cli_neigh6.txt 2>$LOGDIR/fp_cli_neigh6.log
$TIMEOUT fp-cli crypto-lib > $BUGDIR/fp_cli_crypto_lib.txt 2>$LOGDIR/fp_cli_crypto_lib.log
for cryptolib in $(cat $BUGDIR/fp_cli_crypto_lib.txt | grep -v "Available crypto"); do
$TIMEOUT fp-cli stats-crypto $cryptolib > $BUGDIR/fp_cli_stats_crypto_$cryptolib.txt \
2>$LOGDIR/fp_cli_stats_crypto_$cryptolib.log
done
$TIMEOUT fp-cli crypto-offload-stats > $BUGDIR/fp_cli_crypto_offload_stats.txt \
2>$LOGDIR/fp_cli_crypto_offload_stats.log
# bridge
$TIMEOUT fp-cli bridge > $BUGDIR/fp_cli_bridge.txt 2>$LOGDIR/fp_cli_bridge.log
$TIMEOUT fp-cli fp-vswitch-flows > $BUGDIR/fp_cli_fpvs_flows.txt 2>$LOGDIR/fp_cli_fpvs_flows.log
$TIMEOUT fp-cli fp-vswitch-ports > $BUGDIR/fp_cli_fpvs_ports.txt 2>$LOGDIR/fp_cli_fpvs_ports.log
$TIMEOUT fp-cli fp-vswitch-stats > $BUGDIR/fp_cli_fpvs_stats.txt 2>$LOGDIR/fp_cli_fpvs_stats.log
$TIMEOUT fp-cli fp-vswitch-hash-stats > $BUGDIR/fp_cli_fpvs_hash_stats.txt 2>$LOGDIR/fp_cli_fpvs_hash_stats.log
# Netfilter
$TIMEOUT fp-cli filter-bridge broute all > $BUGDIR/fp_cli_ebtables_broute.txt 2>$LOGDIR/fp_cli_ebtables_broute.log
$TIMEOUT fp-cli filter-bridge filter all > $BUGDIR/fp_cli_ebtables_filter.txt 2>$LOGDIR/fp_cli_ebtables_filter.log
# License
if [ -n "$(fp-cli conf compiled | grep CONFIG_MCORE_LICENSE=y)" ]; then
$TIMEOUT fp-cli license > $BUGDIR/fp_cli_license.txt 2>$LOGDIR/fp_cli_license.log
fi
# Log / Lag
$TIMEOUT fp-cli log > $BUGDIR/fp_cli_log.txt 2>$LOGDIR/fp_cli_log.log
$TIMEOUT fp-cli lag > $BUGDIR/fp_cli_lag.txt 2>$LOGDIR/fp_cli_lag.log
# shmem-ports
exec_cmd fp-shmem-ports -d -e all > $BUGDIR/fp_shmem_ports_d.txt 2>$LOGDIR/fp_shmem_ports_d.log
exec_cmd fp-shmem-ports -s -e all > $BUGDIR/fp_shmem_ports_s.txt 2>$LOGDIR/fp_shmem_ports_s.log
if [ -n "$(fp-cli conf compiled | grep CONFIG_MCORE_DPVI=y)" ]; then
exec_cmd fp-shmem-dpvi > $BUGDIR/fp_shmem_dpvi.txt 2>$LOGDIR/fp_shmem_dpvi.log
fi
exec_cmd fp-cpu-usage > $BUGDIR/fp_cpu_usage.txt 2>$LOGDIR/fp_cpu_usage.log
# dpvi
[ -f "/proc/sys/dpvi_shmem/list_shm" ] && \
cp /proc/sys/dpvi_shmem/list_shm $BUGDIR/list_shm.txt
[ -f "/proc/sys/dpvi/list_interfaces" ] && \
cp /proc/sys/dpvi/list_interfaces $BUGDIR/dpvi_list_interfaces.txt
[ -f "/proc/sys/dpvi/running_fastpath" ] && \
cp /proc/sys/dpvi/running_fastpath $BUGDIR/dpvi_running_fastpath.txt
exec_cmd fp-shmem-dpvi > $BUGDIR/fp_shmem_dpvi.txt
# npf
if [ -x "$(command -v fp-npfctl)" ]; then
exec_cmd fp-npfctl vrf-exec all show > $BUGDIR/fp_npf_conf.txt 2>&1
exec_cmd fp-npfctl stats > $BUGDIR/fp_npf_stats.txt 2>&1
exec_cmd fp-npfctl show-params > $BUGDIR/fp_npf_params.txt 2>&1
exec_cmd fp-npfctl pool-usage > $BUGDIR/fp_npf_pool_usage.txt 2>&1
exec_cmd fp-npfctl htable-stats > $BUGDIR/fp_npf_htable_stats.txt 2>&1
exec_cmd fp-npfctl vrf-exec all cgnat-rule all conntrack-stats > $BUGDIR/fp_cgnat_conn_stats.txt 2>&1
exec_cmd fp-npfctl vrf-exec all cgnat-rule all block-stats > $BUGDIR/fp_cgnat_block_stats.txt 2>&1
exec_cmd fp-npfctl vrf-exec all cgnat-rule all port-stats > $BUGDIR/fp_cgnat_port_stats.txt 2>&1
exec_cmd fp-npfctl vrf-exec all cgnat-pool all ip-stats > $BUGDIR/fp_cgnat_ip_stats.txt 2>&1
fi
fi
}
get_fp_info()
{
fp_status=`fast-path.sh status | grep fp-rte | cut -d" " -f 2`
if [ "$fp_status" = 'ok' ]; then
name=$1
if [ -z "$name" ]; then
name=main
cmd="fp-cli"
else
! exec_cmd fp-cli conf compiled | grep -q 'CONFIG_MCORE_VRF=y' && return
idx=`vrfctl list vrfname $name | cut -d" " -f 1 | tr -dc '0-9'`
cmd="fp-cli vrf-exec $idx"
fi
$TIMEOUT $cmd route4 type all > $BUGDIR/$name-fp_cli_route4.txt 2>$LOGDIR/$name-fp_cli_route4.log
$TIMEOUT $cmd route6 type all > $BUGDIR/$name-fp_cli_route6.txt 2>$LOGDIR/$name-fp_cli_route6.log
$TIMEOUT $cmd pbr4-rule > $BUGDIR/$name-fp_cli_pbr4_rule.txt 2>$LOGDIR/$name-fp_cli_pbr4_rule.log
$TIMEOUT $cmd pbr6-rule > $BUGDIR/$name-fp_cli_pbr6_rule.txt 2>$LOGDIR/$name-fp_cli_pbr6_rule.log
# Netfilter
$TIMEOUT $cmd nf4-table filter all > $BUGDIR/$name-fp_cli_nf4_table_filter.txt \
2>$LOGDIR/$name-fp_cli_nf4_table_filter.log
$TIMEOUT $cmd nf4-table mangle all > $BUGDIR/$name-fp_cli_nf4_table_mangle.txt \
2>$LOGDIR/$name-fp_cli_nf4_table_mangle.log
$TIMEOUT $cmd nf4-table nat all > $BUGDIR/$name-fp_cli_nf4_table_nat.txt \
2>$LOGDIR/$name-fp_cli_nf4_table_nat.log
$TIMEOUT $cmd nf4-rules filter > $BUGDIR/$name-fp_cli_nf4_rules_filter.txt \
2>$LOGDIR/$name-fp_cli_nf4_rules_filter.log
$TIMEOUT $cmd nf4-rules mangle > $BUGDIR/$name-fp_cli_nf4_rules_mangle.txt \
2>$LOGDIR/$name-fp_cli_nf4_rules_mangle.log
$TIMEOUT $cmd nf4-rules nat > $BUGDIR/$name-fp_cli_nf4_rules_nat.txt \
2>$LOGDIR/$name-fp_cli_nf4_rules_nat.log
$TIMEOUT $cmd nfct4 summary > $BUGDIR/$name-fp_cli_nf4_ct.txt \
2>$LOGDIR/$name-fp_cli_nf4_ct.log
$TIMEOUT $cmd nf6-table filter all > $BUGDIR/$name-fp_cli_nf6_table_filter.txt \
2>$LOGDIR/$name-fp_cli_nf6_table_filter.log
$TIMEOUT $cmd nf6-table mangle all > $BUGDIR/$name-fp_cli_nf6_table_mangle.txt \
2>$LOGDIR/$name-fp_cli_nf6_table_mangle.log
$TIMEOUT $cmd nf6-rules filter > $BUGDIR/$name-fp_cli_nf6_rules_filter.txt \
2>$LOGDIR/$name-fp_cli_nf6_rules_filter.log
$TIMEOUT $cmd nf6-rules mangle > $BUGDIR/$name-fp_cli_nf6_rules_mangle.txt \
2>$LOGDIR/$name-fp_cli_nf6_rules_mangle.log
$TIMEOUT $cmd nfct6 summary > $BUGDIR/$name-fp_cli_nf6_ct.txt \
2>$LOGDIR/$name-fp_cli_nf6_ct.log
$TIMEOUT $cmd nf-ipset > $BUGDIR/$name-fp_cli_nf_ipset.txt \
2>$LOGDIR/$name-fp_cli_nf_ipset.log
# IPsec
$TIMEOUT $cmd ipsec4-spd all > $BUGDIR/$name-fp_cli_ipsec4_spd_all.txt \
2>$LOGDIR/$name-fp_cli_ipsec4_spd_all.log
$TIMEOUT $cmd ipsec4-sad all > $BUGDIR/$name-fp_cli_ipsec4_sad_all.txt \
2>$LOGDIR/$name-fp_cli_ipsec4_sad_all.log
$TIMEOUT $cmd ipsec6-spd all > $BUGDIR/$name-fp_cli_ipsec6_spd_all.txt \
2>$LOGDIR/$name-fp_cli_ipsec6_spd_all.log
$TIMEOUT $cmd ipsec6-sad all > $BUGDIR/$name-fp_cli_ipsec6_sad_all.txt \
2>$LOGDIR/$name-fp_cli_ipsec6_sad_all.log
$TIMEOUT $cmd xfrmi all > $BUGDIR/$name-fp_cli_xfrmi_all.txt \
2>$LOGDIR/$name-fp_cli_xfrmi_all.log
fi
}
get_openstack_info()
{
# Try setting Openstack if the user forgot telling it in arguments.
# Only working when Openstack processes are still running.
if [ -n "$(ps auxww | grep nova-compute | grep -v 'grep')" ]; then
COMPUTE=yes
OPENSTACK=yes
fi
if [ -n "$(ps auxww | grep nova-api | grep -v 'grep')" ]; then
CONTROLLER=yes
OPENSTACK=yes
fi
if [ -n "$(ps auxww | grep neutron-server | grep -v 'grep')" ]; then
NETWORK=yes
OPENSTACK=yes
fi
[ "$OPENSTACK" != 'yes' ] && return
[ -d "/etc/nova" ] && cp -r /etc/nova $BUGDIR/etc_nova
[ -d "/etc/neutron" ] && cp -r /etc/neutron $BUGDIR/etc_neutron
[ -d "/var/log/nova" ] && cp -r /var/log/nova $BUGDIR/log_nova
[ -d "/var/log/neutron" ] && cp -r /var/log/neutron $BUGDIR/log_neutron
if [ "$CONTROLLER" = 'yes' ]; then
# Check whether OpenStack credentials are correctly set
failure=$($TIMEOUT nova list 2>&1)
if [ "$?" -ne "0" ]; then
printf "\`nova list\` failed with this error: ${failure}\n" >>$LOGDIR/nova_list.log
printf "Make sure your credentials are properly exported, \
and re-run the script.\n" >>$LOGDIR/nova_list.log
return
fi
$TIMEOUT nova --version > $BUGDIR/nova_version.txt 2>&1
$TIMEOUT nova list > $BUGDIR/nova_list.txt 2>>$LOGDIR/nova_list.log
$TIMEOUT nova host-list > $BUGDIR/nova_host_list.txt 2>$LOGDIR/nova_host_list.log
$TIMEOUT nova hypervisor-list > $BUGDIR/nova_hypervisor_list.txt 2>$LOGDIR/nova_hypervisor_list.log
$TIMEOUT nova flavor-list > $BUGDIR/nova_flavor_list.txt 2>LOGDIR/nova_flavor_list.log
$TIMEOUT nova image-list > $BUGDIR/nova_image_list.txt 2>$LOGDIR/nova_image_list.log
$TIMEOUT nova network-list > $BUGDIR/nova_network_list.txt 2>$LOGDIR/nova_network_list.log
$TIMEOUT nova service-list > $BUGDIR/nova_service_list.txt 2>$LOGDIR/nova_service_list.log
for id in $($TIMEOUT nova list --minimal | tail -n +4 | awk '{ print $2 }'); do
[ -n '${id}' ] &&
printf "##### ${id} #####\n" >> $BUGDIR/nova_show.txt &&
$TIMEOUT nova show "${id}" >> $BUGDIR/nova_show.txt
done
fi
if [ "$NETWORK" = 'yes' ]; then
$TIMEOUT neutron agent-list > $BUGDIR/neutron_agent_list.txt 2>$LOGDIR/neutron_agent_list.log
$TIMEOUT neutron router-list > $BUGDIR/neutron_router_list.txt 2>$LOGDIR/neutron_router_list.log
$TIMEOUT neutron net-list > $BUGDIR/neutron_net_list.txt 2>$LOGDIR/neutron_net_list.log
$TIMEOUT neutron subnet-list > $BUGDIR/neutron_net_list.txt 2>$LOGDIR/neutron_net_list.log
fi
}
get_management_info()
{
if [ -d /etc/sysrepo ]; then
cp -a /etc/sysrepo $BUGDIR/etc_sysrepo
exec_cmd sysrepocfg -X$BUGDIR/vrouter_startup.json \
-m vrouter -d startup -f json
exec_cmd sysrepocfg -X$BUGDIR/vrouter_running.json \
-m vrouter -d running -f json
fi
}
get_license_info()
{
if command -v vrl-status > /dev/null; then
vrl-status > $BUGDIR/vrl_status.txt
fi
}
get_qemu_libvirt_version()
{
if command -v qemu-kvm > /dev/null; then
qemu-kvm --version > "$BUGDIR/qemu_kvm.version.txt"
fi
if command -v libvirtd > /dev/null; then
libvirtd --version > "$BUGDIR/libvirtd.version.txt"
fi
}
get_distribution_supported()
{
if [ ! -r /etc/os-release ]; then
return
fi
. /etc/os-release
unsupported=""
if uname -a | grep -q "x86_64"; then
case "$ID" in
bclinux)
if [ "$VERSION" != "21.10 (LTS-SP2)" ]; then
unsupported="$PRETTY_NAME not supported"
fi
# Check qemu
expected_version="16:6.2.0-44.9"
dnf list --installed qemu\* | grep qemu > /tmp/qemu_version.txt
while read -r line; do
pkg="$(echo "$line" | awk '{print $1}')"
ver="$(echo "$line" | awk '{print $2}')"
if [ "$ver" != "$expected_version" ]; then
unsupported="$unsupported
Wrong $pkg version, expected $expected_version installed version is $ver"
fi
done < /tmp/qemu_version.txt
[ -e /tmp/qemu_version.txt ] && rm /tmp/qemu_version.txt
# check libvirtd
expected_version="8.0.0-5.7.oe1.bclinux"
dnf list --installed libvirt\* | grep libvirt > /tmp/libvirt_version.txt
while read -r line; do
pkg="$(echo "$line" | awk '{print $1}')"
ver="$(echo "$line" | awk '{print $2}')"
if [ "$ver" != "$expected_version" ]; then
unsupported="$unsupported
Wrong $pkg version, expected $expected_version installed version is $ver"
fi
done < /tmp/libvirt_version.txt
[ -e /tmp/libvirt_version.txt ] && rm /tmp/libvirt_version.txt
;;
esac
fi
if [ -n "$unsupported" ]; then
echo "$unsupported" >> "$BUGDIR/unsupported.txt" 2>&1
printf "\n\nVirtual-Accelerator is not supported on this distribution\n\n"
fi
}
bgp_commands="
show bgp vrf all summary
show ip bgp vrf all
show ip bgp vrf all neighbors
show ip bgp vrf all summary
show ip bgp vrf all statistics
show ip bgp vrf all update-groups advertise-queue
show ip bgp vrf all update-groups advertised-routes
show ip bgp vrf all update-groups packet-queue
show ip bgp vrf all update-groups statistics
show ip bgp vrf all peer-group
show ip bgp memory
show bgp vrf all ipv6
show bgp vrf all ipv6 neighbors
show bgp vrf all ipv6 summary
show bgp vrf all ipv6 update-groups advertise-queue
show bgp vrf all ipv6 update-groups advertised-routes
show bgp vrf all ipv6 update-groups packet-queue
show bgp vrf all ipv6 update-groups statistics
show bgp vrf all martian next-hop
show bgp vrf all nexthop
show bgp evpn route
"
zebra_commands="
show zebra
show zebra client summary
show ip zebra route dump json
show ipv6 zebra route dump json
show ip nht vrf all
show route-map
show memory
show interface vrf all
show vrf
show zebra fpm stats
show error all
show work-queues
show debugging hashtable
show running-config
show thread cpu
show thread poll
show daemons
show version
"
get_frr_info()
{
if [ ! -x "$(command -v vtysh)" ]; then
return
fi
mkdir -p $BUGDIR/frr
printf '%s' "$zebra_commands" |
while read -r command; do
[ -n "$command" ] || continue
echo "$command"
# add title underline
echo "$command" | sed 's/./=/g'
echo
vtysh -c "$command" 2>&1
echo
echo
done > $BUGDIR/frr/zebra.txt
printf '%s' "$bgp_commands" |
while read -r command; do
[ -n "$command" ] || continue
echo "$command"
# add title underline
echo "$command" | sed 's/./=/g'
echo
vtysh -c "$command" 2>&1
echo
echo
done > $BUGDIR/frr/bgp.txt
}
trap cleanup EXIT INT QUIT
TMPDIR=$(mktemp -d)
SUFFIX=$(hostname)
TIMEOUT_S=10
BUGDIR=$TMPDIR/bug_info/$SUFFIX
LOGDIR=$BUGDIR/log
EXTRAFILES=""
mkdir -p $BUGDIR
mkdir -p $LOGDIR
exec 2> $LOGDIR/troubleshooting-report-stderr.log
[ -x "$(command -v timeout)" ] && TIMEOUT="$(command -v timeout) ${TIMEOUT_S}s"
parse_args $@
ARCHIVE=${ARCHIVE:-"/tmp/troubleshooting-report_${SUFFIX}.tar.gz"}
printf 'Gathering information. This may take some time...\n'
# octeon-specific info
[ -f "/proc/octeon_ethernet_stats" ] && \
cp /proc/octeon_ethernet_stats > $BUGDIR/octeon_ethernet_stats.txt
[ -f "/proc/octeon_info" ] && cp /proc/octeon_info > $BUGDIR/octeon_info.txt
get_global_fp_info
if [ -n "$(ip netns)" ]; then
for vrf in $(ip netns | cut -d " " -f 1); do
get_linux_info $vrf
get_fp_info $vrf
done
fi
get_fp_info
get_linux_info
# va-specefic files [SF13646] [SF13686]
[ -f "/etc/fp-vdev-conf.old" ] && cp /etc/fp-vdev-conf.old $BUGDIR/fp-vdev-conf.old
[ -f "/etc/fp-vdev-conf.json" ] && cp /etc/fp-vdev-conf.json $BUGDIR/fp-vdev-conf.json
get_system_info
get_coredump
get_frr_info
get_openstack_info
get_management_info
get_license_info
get_qemu_libvirt_version
get_distribution_supported
# Add extra files
for file in ${EXTRAFILES}; do
cp ${file} $BUGDIR/`basename ${file}`
done
find ${LOGDIR} -maxdepth 1 -type f -empty -delete
find ${BUGDIR} -maxdepth 1 -type f -empty -delete
tar -czf ${ARCHIVE} -C ${TMPDIR} .
printf "Saved into ${ARCHIVE}\n"
Send the troubleshooting-report*.tar.gz files along with any other information
you deem relevant.
Note
The script makes use of the following commands if available:
brctl
dmidecode
ebtables-save
ip6tables-save
iptables-save
lsb_release
lscpu
lspci
lstopo
mount
numastat
ovs-vsctl
sosreport
timeout
vmstat
For better results, make sure these tools are available on your system.