#!/bin/sh # # Author: Mark Lamourine # Copyright 2012 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script checks the status of a host that is running an Openshift Origins # Broker service. # # Check that the broker components are installed and communicating properly # OSO_CONF_DIR=${OSO_CONF_DIR:=/etc/stickshift} OSO_BROKER_ROOT=${OSO_BROKER_ROOT:=/var/www/stickshift/broker} OSO_CFG_DIR=${OSO_BROKER_ROOT}/config OSO_ENV_DIR=${OSO_CFG_DIR}/environments OSO_INI_DIR=${OSO_CFG_DIR}/initializers OSO_ENVIRONMENT=${OSO_ENVIRONMENT:=development} DATA_PLUGINS="mongo" AUTH_PLUGINS="mongo" NAME_PLUGINS="bind" MESG_PLUGINS="oddjob" DATA_MODULE_PATTERN="rubygem-crankcase-%NAME%-plugin" AUTH_MODULE_PATTERN="rubygem-swingshift-%NAME%-plugin" NAME_MODULE_PATTERN="rubygem-uplift-%NAME%-plugin" MESG_MODULE_PATTERN="rubygem-gearchanger-%NAME%-plugin" DEFAULT_DATA_SERVICE="mongo" DEFAULT_AUTH_SERVICE="mongo" DEFAULT_NAME_SERVICE="bind" DEFAULT_MESG_SERVICE="oddjob" DEFAULT_PACKAGES=" ruby rubygems rubygem-rails rubygem-passenger rubygem-stickshift-common rubygem-stickshift-controller stickshift-broker " DEFAULT_SERVICES="stickshift-broker" # Depends on OSO_BROKER_ROOT in ruby lib search list OSO_RUBY_LIBS="stickshift-controller config/application" if [ -z "$PACKAGES" ] then PACKAGES="$DEFAULT_PACKAGES" else echo WARNING: ENV overrides PACKAGES >&2 fi if [ -z "$SERVICES" ] then SERVICES="$DEFAULT_SERVICES" else echo WARNING: ENV overrides SERVICES >&2 fi # ============================================================================ # # Utility Functions # # ============================================================================ function verbose() { # MESSAGE=$* if [ -n "$VERBOSE" ] then echo "INFO: $*" fi } function notice() { # MESSAGE=$* echo "NOTICE: $*" } function fail() { # MESSAGE=$* echo "FAIL: $*" >&2 STATUS=$(($STATUS + 1)) } # ========================================================================== # # ========================================================================== function probe_version() { # Find out who owns /var/www/stickshift/broker BROKER_RPM=$(rpm -qf $OSO_BROKER_ROOT --qf '%{NAME}\n' 2>/dev/null) if [ -z "$BROKER_RPM" ] then fail "broker Rails app root ${OSO_BROKER_ROOT} does not exist: no broker package" else verbose "Broker package is: $BROKER_RPM" fi } function check_ruby_requirements() { verbose checking ruby requirements if [ ! -x /usr/bin/ruby ] then fail ruby is not installed or not executable return fi for OSO_RUBY_LIB in $* do verbose checking ruby requirements for $OSO_RUBY_LIB GEM_ERROR=`ruby -I ${OSO_BROKER_ROOT} -r rubygems -- < gem_error puts "Gem Not Found for $OSO_RUBY_LIB: #{gem_error}" exit 1 end EOF` if [ $? -ne 0 ] then fail module $OSO_RUBY_LIB -- "$GEM_ERROR" fi done } # # Get a single value from the Rails application. # Yes, this is inefficient, but the point is to not depend on the system # you're examining (Ruby/Rails) to confirm itself. # function get_application_value() { # $1 = RUBY_VARNAME if [ -z "$OSO_BROKER_ROOT" ] then fail OSO_BROKER_ROOT is unset. Cannot load app configuration exit $STATUS fi if [ -z "$OSO_ENVIRONMENT" ] then fail OSO_ENVIRONMENT is unset. Cannot load app configuration exit $STATUS fi if which ruby 2>/dev/null >/dev/null then ruby -I ${OSO_BROKER_ROOT} -r rubygems -- <&1 require 'bundler' require 'rails' begin require 'stickshift-controller' require 'config/application' require 'config/initializers/broker' require 'config/environments/$OSO_ENVIRONMENT' rescue Bundler::GemNotFound => gem_error puts "Error loading libs or gems: #{gem_error}" exit 1 end puts $1 EOF else fail ruby is not installed or not executable fi } # ============================================================================ # # Base Packages and Components # # ============================================================================ # Network Manager (disabled or not present) # network "service" enabled # SSH max connections # Kernel Settings # - Ephemeral Port Range # - Kernel Semaphores (httpd communications) # - netfilter conntrack buffer size # IPTables # ============================================================================ # # Configuration and Variables # # ============================================================================ # # Check packages # function check_packages() { # $* = $PACKAGES verbose checking packages for PKGNAME in $* do verbose checking package $PKGNAME PKGSTATUS=`rpm -q $PKGNAME` if echo $PKGSTATUS | grep "not installed" >/dev/null then fail "package $PKGNAME is not installed" fi done } # # There are two different service monitors: RHEL6 still uses service and chkconfig # Fedora 16+ use systemctl (as part of systemd) # function service_enabled() { # $1 = SERVICE_NAME if [ -x /bin/systemctl -a ! -x /etc/init.d/$1 ] then systemctl is-enabled $1.service 2>&1 >/dev/null else chkconfig $1 fi if [ $? != 0 ] then fail "service $1 not enabled" fi } function service_running() { # $1 = SERVICE_NAME if [ -x /bin/systemctl ] then systemctl status $1.service 2>&1 | grep -e "^\s*Active: active" >/dev/null else service $1 status 2>&1 > /dev/null fi if [ $? != 0 ] then fail "service $1 not running" fi } # # Check services # function check_services() { verbose checking services for SVCNAME in $SERVICES do service_enabled $SVCNAME service_running $SVCNAME done } function check_selinux_modules() { verbose "checking that selinux modules are loaded" # if not enforcing, just say so notice "SELinux is " `getenforce` if [ `getenforce 2>/dev/null` != "Enforcing" ] then return fi SEMODULE=$(which semodule 2>/dev/null) if [ $? != 0 ] then fail "semodule binary not present (from policycoreutils RPM)" return fi SELINUX_MODULES='rubygem_passenger stickshift-broker' for MODNAME in $SELINUX_MODULES do MODSPEC=$($SEMODULE -l | grep $MODNAME) if [ -n "$MODSPEC" ] then MODVERS=$(echo $MODSPEC | awk '{print $2;}') verbose "selinux module $MODNAME is version $MODVERS" else fail "module $MODNAME is not loaded" fi done } function check_kernel_settings() { verbose "checking kernel settings" KERNEL_SETTINGS="kernel.sem net.ipv4.ip_local_port_range net.netfilter.nf_conntrack_max" for KERNEL_KEY in $KERNEL_SETTINGS do notice current - $(sysctl $KERNEL_KEY) notice config - $(grep -e "^$KERNEL_KEY " /etc/sysctl.conf) done } function check_firewall() { verbose "checking firewall settings" # check for 'enabled' service_enabled iptables service_running iptables # Need to check for ports TCP/22, TCP/53, TCP/80, TCP/443 and UDP/53 allowed } # # Check SELinux # function check_selinux_booleans() { # if not enforcing, just say so notice "SELinux is " `getenforce` if [ `getenforce 2>/dev/null` != "Enforcing" ] then return fi # check httpd_unified if getsebool httpd_unified | grep -e '--> on' >/dev/null then verbose SELinux boolean httpd_unified is enabled else fail SELinux boolean httpd_unified is disabled fi # Only needed with BIND DDNS plugin # check allow_ypbind # check httpd_unified if getsebool allow_ypbind | grep -e '--> on' >/dev/null then verbose SELinux boolean allow_ypbind is enabled else notice SELinux boolean allow_ypbind is disabled fi } # ============================================================================ # # DataStore # # ============================================================================ function find_datastore_plugin() { # StickShift::DataStore.instance_variable_get('@ss_ds_provider') get_application_value "StickShift::DataStore.instance_variable_get('@ss_ds_provider')" } function check_datastore() { verbose checking datastore DATA_PLUGIN=`find_datastore_plugin` case $DATA_PLUGIN in 'StickShift::DataStore') fail abstract datastore class: $DATA_PLUGIN ;; 'StickShift::MongoDataStore') verbose "datastore plugin: $DATA_PLUGIN" check_datastore_mongo ;; *) fail unknown datastore class: $DATA_PLUGIN ;; esac unset DATA_PLUGIN } function check_mongo_login() { # $HOST_PORT=$1 # $DB=$2 # $USER=$3 # $PASS=$4 verbose checking mongo db login access mongo $1/$2 --username $3 --password <&1 >/dev/null $4 exit EOF if [ $? -eq 0 ] then verbose "mongo db login successful: $1/$2 --username $3" else fail "error logging into mongo db: $1/$2 --username $3, exit code: $?" fi } function check_datastore_mongo() { verbose checking mongo datastore configuration # get the service hostname, port, username, password DS_HOST=`get_application_value "Rails.application.config.datastore[:host_port][0]"` DS_PORT=`get_application_value "Rails.application.config.datastore[:host_port][1]"` DS_USER=`get_application_value "Rails.application.config.datastore[:user]"` DS_PASS=`get_application_value "Rails.application.config.datastore[:password]"` DS_NAME=`get_application_value "Rails.application.config.datastore[:db]"` verbose "Datastore Host: $DS_HOST" verbose "Datastore Port: $DS_PORT" verbose "Datastore User: $DS_USER" if [ "$DS_PASS" == "mooo" ] then fail "Datastore Password is still the default" else verbose "Datastore Password has been reset" fi verbose "Datastore DB Name: $DS_NAME" # Only check local values if DS_HOST is localhost if [ "$DS_HOST" = "localhost" ] then verbose "Datastore configuration is on localhost" # check presence of mongodb package check_packages mongodb # check auth enabled if grep -e '^auth = true' /etc/mongodb.conf >/dev/null then verbose "LOCAL: mongod auth is enabled" else fail "LOCAL: mongod auth is not enabled in /etc/mongodb.conf" fi # check service enabled if (service mongod status 2>/dev/null | grep enabled 2>&1 >/dev/null) then verbose "LOCAL: mongod service enabled" else fail "LOCAL: mongod service not enabled" fi # check service started if (service mongod status 2>/dev/null | grep 'active (running)' 2>&1 >/dev/null) then verbose "LOCAL: mongod service running" else fail "LOCAL: mongod service not running" fi else verbose "Datastore: mongo db service is remote" fi # check stickshift user (username/password) check_mongo_login $DS_HOST:$DS_PORT $DS_NAME $DS_USER $DS_PASS } function check_auth_mongo() { verbose checking mongo auth configuration # get the service hostname, port, username, password DS_HOST=`get_application_value "Rails.application.config.auth[:mongo_host_port][0]"` DS_PORT=`get_application_value "Rails.application.config.auth[:mongo_host_port][1]"` DS_USER=`get_application_value "Rails.application.config.auth[:mongo_user]"` DS_PASS=`get_application_value "Rails.application.config.auth[:mongo_password]"` DS_NAME=`get_application_value "Rails.application.config.auth[:mongo_db]"` verbose "Auth Host: $DS_HOST" verbose "Auth Port: $DS_PORT" verbose "Auth User: $DS_USER" if [ "$DS_PASS" == "mooo" ] then fail "Datastore Password is still the default" else verbose "Datastore Password has been reset" fi verbose "Auth DB Name: $DS_NAME" # Only check local values if DS_HOST is localhost if [ "$DS_HOST" = "localhost" ] then verbose "Auth configuration is on localhost" # check presence of mongodb package check_packages mongodb # check auth enabled if grep -e '^auth = true' /etc/mongodb.conf >/dev/null then verbose "LOCAL: mongod auth is enabled" else fail "LOCAL: mongod auth is not enabled in /etc/mongodb.conf" fi # check service enabled if (service mongod status 2>/dev/null | grep enabled 2>&1 >/dev/null) then verbose "LOCAL: mongod service enabled" else fail "LOCAL: mongod service not enabled" fi # check service started if (service mongod status 2>/dev/null | grep 'active (running)' 2>&1 >/dev/null) then verbose "LOCAL: mongod service running" else fail "LOCAL: mongod service not running" fi else verbose "Auth: mongo db service is remote" fi # check stickshift user (username/password) check_mongo_login $DS_HOST:$DS_PORT $DS_NAME $DS_USER $DS_PASS } # ============================================================================ # # Cloud User Authentication # # ============================================================================ function find_authentication_plugin() { # StickShift::AuthService.instance_variable_get('@ss_auth_provider') get_application_value "StickShift::AuthService.instance_variable_get('@ss_auth_provider')" } function check_authentication() { verbose checking cloud user authentication AUTH_PLUGIN=`find_authentication_plugin` verbose auth plugin = $AUTH_PLUGIN case $AUTH_PLUGIN in 'StickShift::AuthService') fail abstract auth class: $AUTH_PLUGIN ;; 'Swingshift::MongoAuthService') verbose "auth plugin: $AUTH_PLUGIN" check_auth_mongo ;; *) fail unknown auth class: $AUTH_PLUGIN ;; esac unset AUTH_PLUGIN } # ============================================================================ # # Dynamic DNS Updates # # ============================================================================ function find_dynamic_dns_plugin() { # StickShift::DnsService.instance_variable_get('@ss_dns_provider') get_application_value "StickShift::DnsService.instance_variable_get('@ss_dns_provider')" } function dns_bind_update_record() { # $1 = server # $2 = key name # $3 = key value # $4 = function (add|delete) # $5 = type (A, TXT, CNAME) # $6 = name # $7 = value # check $1: should be an IP address # check $3: should be a key string # check $4: should be add|delete # check $5 should be A, TXT, CNAME verbose "${4}ing $5 record named $6 to server $1: $7" nsupdate </dev/null then verbose "txt record successfully added" else fail "txt record testrecord.$DNS_SUFFIX does not resolve on server $DNS_SERVER" fi # remove it. dns_bind_update_record $DNS_SERVER $DNS_KEYNAME $DNS_KEYVAL delete txt testrecord.$DNS_SUFFIX # verify that the record is removed if host -t txt testrecord.$DNS_SUFFIX $DNS_SERVER >/dev/null then fail "txt record testrecord.$DNS_SUFFIX still resolves on server $DNS_SERVER" else verbose "txt record successfully deleted" fi } function check_dynamic_dns() { verbose checking dynamic dns plugin NAME_PLUGIN=`find_dynamic_dns_plugin` case $NAME_PLUGIN in 'StickShift::DnsService') fail abstract dns class: $NAME_PLUGIN ;; 'Uplift::BindPlugin') verbose dynamic dns plugin = $NAME_PLUGIN check_dns_bind ;; *) fail unknown dns class: $NAME_PLUGIN ;; esac unset NAME_PLUGIN } # ============================================================================ # # Broker -> Node messaging # # ============================================================================ function find_messaging_plugin() { # StickShift::ApplicationContainerProxy.instance_variable_get('@proxy_provider') get_application_value "StickShift::ApplicationContainerProxy.instance_variable_get('@proxy_provider')" } function check_messaging_oddjob() { verbose checking oddjob messaging # config files # # } #function send_oddjob_message() { #ruby -I ${OSO_BROKER_ROOT} -r rubygems -- <&1 # require 'stickshift-controller' # require 'gearchanger-oddjob-plugin' #EOF #} function check_messaging() { verbose checking messaging configuration MSG_PLUGIN=`find_messaging_plugin` case $MSG_PLUGIN in 'StickShift::ApplicationContainerProxy') fail abstract messaging class: $MSG_PLUGIN ;; 'GearChanger::OddJobApplicationContainerProxy') verbose messaging plugin = $MSG_PLUGIN check_messaging_oddjob ;; *) fail unknown messaging class: $MSG_PLUGIN ;; esac unset MSG_PLUGIN } function check_node_communications() { sudo -u apache ruby - <