#!/bin/bash

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or

# implied.

#

# See the License for the specific language governing permissions and

# limitations under the License.

export PATH=$PATH:/usr/local/sbin:/usr/sbin

# This script attempts to recover as much generic diagnostic logs as it can

LOGROOT=${WORKSPACE:-/tmp}

LOGDIR="${LOGROOT}/logs"

DIAG_LOGDIR="${LOGDIR}/diag"

CONF_LOGDIR="${LOGDIR}/etc"

GIT_URL="http://git.openstack.org/cgit"

PROJECTS_URL="${GIT_URL}/openstack/governance/plain/reference/projects.yaml"

if [ $(id -u) != 0 ]; then

SUDO='sudo'

fi

$SUDO mkdir -p "${DIAG_LOGDIR}"

$SUDO mkdir -p "${CONF_LOGDIR}"

function get_diag_commands {

echo "Setting up list of diagnostic commands to run..."

commands=(

'df -h'

'dmesg -T'

'fdisk -l'

'lsmod'

'iptables -vnL'

'iptables -vnL -t nat'

'iptables -vnL -t mangle'

'ifconfig -a'

'ip addr show'

'lsmod'

'timeout 15 lsof' # High potential of getting stuck

'lsof -Pni'

'netstat -ntlp'

'pstree -p'

'sysctl -a'

'yum repolist -v'

'rpm -qa'

'journalctl --no-pager'

'ulimit -n'

)

echo "Installing required RPM packages..."

$SUDO yum -y install coreutils curl file lsof net-tools psmisc

echo "Running diagnostic commands..."

for ((i = 0; i < ${#commands[@]}; i++)); do

# filenames have underscores instead of spaces or slashes

filename="$(echo "${commands[$i]}" |sed -e "s%[ \/]%_%g").txt"

# Run diagnostic commands but don't fail the whole thing if one command fails

$SUDO bash -c "${commands[$i]} 2>&1 > ${DIAG_LOGDIR}/${filename}" || true

done

}

function get_config_and_logs {

echo "Setting up and discovering directories and files to recover..."

# Paths we're interested in

paths=(

'/proc/cpuinfo'

'/proc/meminfo'

'/proc/mounts'

'/etc/os-release'

'/etc/sudoers'

'/etc/sudoers.d'

'/etc/hosts'

'/etc/pip.conf'

'/etc/libvirt'

'/var/log/libvirt'

'/etc/qemu'

'/etc/qemu-kvm'

'/etc/openvswitch'

'/var/log/openvswitch'

'/etc/openstack-dashboard'

'/etc/aodh' # aodh is nested under telemetry in governance

'/var/log/aodh'

'/etc/ceilometer' # ceilometer is nested under telemetry in governance

'/var/log/ceilometer'

'/etc/gnocchi' # gnocchi is nested under telemetry in governance

'/var/log/gnocchi'

'/etc/rabbitmq/'

'/var/log/rabbitmq'

'/etc/my.cnf.d'

'/var/log/mariadb'

'/etc/httpd/conf.d/'

'/var/log/httpd'

'/var/tmp/packstack/latest'

'/var/tmp/packstack/latest/testrepository.subunit' # So we're copying it

'/var/log/audit' # to the root of

'/var/log/secure' # /logs

'/var/log/messages'

'/var/log/dstat.log'

'/etc/puppet/puppet.conf'

'/etc/puppet/hiera.yaml'

)

# Add discovered project directories from official governance

# Optimistic parsing.. find a better way

project_list=$(curl $PROJECTS_URL 2>&1 | egrep "^\S+:$" |cut -f1 -d :)

for project in $project_list; do

paths+=("/etc/${project}")

paths+=("/var/log/${project}")

done

echo "Recovering files and directories..."

# Send things to appropriate log directories if they exist

for ((i = 0; i < ${#paths[@]}; i++)); do

if [ -e ${paths[$i]} ]; then

if [[ "${paths[$i]}" =~ /proc/ ]]; then

$SUDO cp "${paths[$i]}" ${DIAG_LOGDIR}/

elif [[ "${paths[$i]}" =~ /var/ ]]; then

$SUDO cp -Lr "${paths[$i]}" ${LOGDIR}/

elif [[ "${paths[$i]}" =~ /etc/ ]]; then

$SUDO cp -Lr "${paths[$i]}" ${CONF_LOGDIR}/

fi

fi

done

}

function ensure_log_properties {

echo "Making sure directories and files have the right properties..."

FIND="${SUDO} find ${LOGDIR} ! -path '*.git/*'"

# Ensure files are readable by everyone

$FIND -type d -execdir $SUDO chmod 755 '{}' \;

$FIND -type f -execdir $SUDO chmod 644 '{}' \;

# Ensure files are in .txt when possible (web mime type)

for file in $($FIND -type f ! -name "*.txt"); do

if [[ "$(file --mime-type ${file} |cut -f2 -d :)" =~ text ]]; then

$SUDO mv ${file} ${file}.txt

fi

done

echo "Compressing all text files..."

# Compress all files

$FIND -iname '*.txt' -execdir gzip -f -9 {} \+

echo "Compressed log and configuration can be found in ${LOGDIR}."

}

function recover_default_logs {

get_diag_commands

get_config_and_logs

ensure_log_properties

}