Ensure we install setuptools from requirements

Use pip_install_gr so we get the version pinned by requirements.  The
depends-on is an example of where we're trying to pin to workaround
issues.

Depends-On: I9c57c08a150571c5bb62235d502839394d53a4c1
Change-Id: I780cca681b12a3e9d228dbf2fd9fa6e8ab1a82e1
(cherry picked from commit 1e7f738f28)
(cherry picked from commit 0d2fc817ea)

.gitreview

@@ -2,3 +2,4 @@
 host=review.openstack.org
 port=29418
 project=openstack-dev/devstack.git
+defaultbranch=stable/mitaka

extras.d/80-tempest.sh

@@ -13,6 +13,8 @@ if is_service_enabled tempest; then
     elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
         echo_summary "Initializing Tempest"
         configure_tempest
+        echo_summary "Installing Tempest Plugins"
+        install_tempest_plugins
     elif [[ "$1" == "stack" && "$2" == "post-extra" ]]; then
         # local.conf Tempest option overrides
         :

files/debs/general

@@ -1,5 +1,6 @@
 bc
 bridge-utils
+bsdmainutils
 curl
 g++
 gcc

files/rpms-suse/general

@@ -25,5 +25,6 @@ screen
 tar
 tcpdump
 unzip
+util-linux
 wget
 zlib-devel

files/rpms/general

@@ -30,6 +30,7 @@ screen
 tar
 tcpdump
 unzip
+util-linux
 wget
 which
 zlib-devel

functions

@@ -30,6 +30,19 @@ function function_exists {
     declare -f -F $1 > /dev/null
 }
 
+# short_source prints out the current location of the caller in a way
+# that strips redundant directories. This is useful for PS4 usage.
+function short_source {
+    saveIFS=$IFS
+    IFS=" "
+    called=($(caller 0))
+    IFS=$saveIFS
+    file=${called[2]}
+    file=${file#$RC_DIR/}
+    printf "%-40s " "$file:${called[1]}:${called[0]}"
+}
+
+
 # Retrieve an image from a URL and upload into Glance.
 # Uses the following variables:
 #
@@ -655,6 +668,15 @@ function create_disk {
     fi
 }
 
+
+# set_mtu - Set MTU on a device
+function set_mtu {
+    local dev=$1
+    local mtu=$2
+    sudo ip link set mtu $mtu dev $dev
+}
+
+
 # Restore xtrace
 $_XTRACE_FUNCTIONS
 

inc/ini-config

@@ -274,6 +274,170 @@ function iniget_sections {
     $xtrace
 }
 
+# Set a localrc var
+function localrc_set {
+    local file=$1
+    local group="local"
+    local conf="localrc"
+    local section=""
+    local option=$2
+    local value=$3
+    localconf_set "$file" "$group" "$conf" "$section" "$option" "$value"
+}
+
+# Check if local.conf has section.
+function localconf_has_section {
+    local file=$1
+    local group=$2
+    local conf=$3
+    local section=$4
+    local sep
+    sep=$(echo -ne "\x01")
+    local line
+    line=$(sed -ne "\\${sep}^\[\[${group}|${conf}\]\]${sep},\\${sep}\[\[.*\]\]${sep}{
+        /\[${section}\]/p
+    }" "$file")
+    [ -n "$line" ]
+}
+
+# Check if local.conf has option.
+function localconf_has_option {
+    local file=$1
+    local group=$2
+    local conf=$3
+    local section=$4
+    local option=$5
+    local sep
+    sep=$(echo -ne "\x01")
+    local line
+    if [[ -z "$section" ]]; then
+        line=$(sed -ne "\\${sep}^\[\[${group}|${conf}\]\]${sep},\\${sep}\[\[.*\]\]${sep}{
+            /${option}[ \t]*=.*$/p
+        }" "$file")
+    else
+        line=$(sed -ne "\\${sep}^\[\[${group}|${conf}\]\]${sep},\\${sep}\[\[.*\]\]${sep}{
+            /\[${section}\]/,/\[\[.*\]\]\|\[.*\]/{
+                /${option}[ \t]*=.*$/p}
+        }" "$file")
+    fi
+    [ -n "$line" ]
+}
+
+# Update option in local.conf.
+function localconf_update_option {
+    local sudo=$1
+    local file=$2
+    local group=$3
+    local conf=$4
+    local section=$5
+    local option=$6
+    local value=$7
+    local sep
+    sep=$(echo -ne "\x01")
+    if [[ -z "$section" ]]; then
+        $sudo sed -i -e "\\${sep}^\[\[${group}|${conf}\]\]${sep},\\${sep}\[\[.*\]\]${sep}{
+            s${sep}^\(${option}[ \t]*=[ \t]*\).*\$${sep}\1${value}${sep}
+        }" "$file"
+    else
+        $sudo sed -i -e "\\${sep}^\[\[${group}|${conf}\]\]${sep},\\${sep}\[\[.*\]\]${sep}{
+            /\[${section}\]/,/\[\[.*\]\]\|\[.*\]/s${sep}^\(${option}[ \t]*=[ \t]*\).*\$${sep}\1${value}${sep}
+        }" "$file"
+    fi
+}
+
+# Add option in local.conf.
+function localconf_add_option {
+    local sudo=$1
+    local file=$2
+    local group=$3
+    local conf=$4
+    local section=$5
+    local option=$6
+    local value=$7
+    local sep
+    sep=$(echo -ne "\x01")
+    if [[ -z "$section" ]]; then
+        $sudo sed -i -e "\\${sep}^\[\[${group}|${conf}\]\]${sep} a $option=$value" "$file"
+    else
+        $sudo sed -i -e "\\${sep}^\[\[${group}|${conf}\]\]${sep},\\${sep}\[\[.*\]\]${sep}{
+            /\[${section}\]/ a $option=$value
+        }" "$file"
+    fi
+}
+
+# Add section and option in local.conf.
+function localconf_add_section_and_option {
+    local sudo=$1
+    local file=$2
+    local group=$3
+    local conf=$4
+    local section=$5
+    local option=$6
+    local value=$7
+    local sep
+    sep=$(echo -ne "\x01")
+    $sudo sed -i -e "\\${sep}^\[\[${group}|${conf}\]\]${sep} {
+        a [$section]
+        a $option=$value
+    }" "$file"
+}
+
+# Set an option in a local.conf file.
+# localconf_set [-sudo] config-file group conf-name section option value
+#  - if the file does not exist, it is created
+function localconf_set {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local sep
+    sep=$(echo -ne "\x01")
+    local sudo=""
+    if [ $1 == "-sudo" ]; then
+        sudo="sudo "
+        shift
+    fi
+    local file=$1
+    local group=$2
+    local conf=$3
+    local section=$4
+    local option=$5
+    local value=$6
+
+    if [[ -z $group || -z $conf || -z $option || -z $value ]]; then
+        $xtrace
+        return
+    fi
+
+    if ! grep -q "^\[\[${group}|${conf}\]\]" "$file" 2>/dev/null; then
+        # Add meta section at the end if it does not exist
+        echo -e "\n[[${group}|${conf}]]" | $sudo tee --append "$file" > /dev/null
+        # Add section at the end
+        if [[ -n "$section" ]]; then
+            echo -e "[$section]" | $sudo tee --append "$file" > /dev/null
+        fi
+        # Add option at the end
+        echo -e "$option=$value" | $sudo tee --append "$file" > /dev/null
+    elif [[ -z "$section" ]]; then
+        if ! localconf_has_option "$file" "$group" "$conf" "$section" "$option"; then
+            # Add option
+            localconf_add_option "$sudo" "$file" "$group" "$conf" "$section" "$option" "$value"
+        else
+            # Replace it
+            localconf_update_option "$sudo" "$file" "$group" "$conf" "$section" "$option" "$value"
+        fi
+    elif ! localconf_has_section "$file" "$group" "$conf" "$section"; then
+        # Add section and option in specified meta section
+        localconf_add_section_and_option "$sudo" "$file" "$group" "$conf" "$section" "$option" "$value"
+    elif ! localconf_has_option "$file" "$group" "$conf" "$section" "$option"; then
+        # Add option
+        localconf_add_option "$sudo" "$file" "$group" "$conf" "$section" "$option" "$value"
+    else
+        # Replace it
+        localconf_update_option "$sudo" "$file" "$group" "$conf" "$section" "$option" "$value"
+    fi
+    $xtrace
+}
+
 # Restore xtrace
 $INC_CONF_TRACE
 

inc/python

@@ -148,11 +148,15 @@ function pip_install {
     fi
 
     $xtrace
+    # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
+    # the same behaviour of setuptools before version 25.0.0.
+    # related issue: https://github.com/pypa/pip/issues/3874
     $sudo_pip \
         http_proxy="${http_proxy:-}" \
         https_proxy="${https_proxy:-}" \
         no_proxy="${no_proxy:-}" \
         PIP_FIND_LINKS=$PIP_FIND_LINKS \
+        SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
         $cmd_pip $upgrade \
         $@
     result=$?
@@ -363,6 +367,13 @@ function install_python3 {
     fi
 }
 
+function install_devstack_tools {
+    # intentionally old to ensure devstack-gate has control
+    local dstools_version=${DSTOOLS_VERSION:-0.1.2}
+    install_python3
+    sudo pip3 install -U devstack-tools==${dstools_version}
+}
+
 # Restore xtrace
 $INC_PY_TRACE
 

lib/dlm

@@ -93,6 +93,8 @@ function install_dlm {
     if is_dlm_enabled; then
         if is_ubuntu; then
             install_package zookeeperd
+        elif is_fedora; then
+            install_package zookeeper
         else
             die $LINENO "Don't know how to install zookeeper on this platform"
         fi

lib/heat

@@ -40,7 +40,6 @@ HEAT_DIR=$DEST/heat
 HEAT_CFNTOOLS_DIR=$DEST/heat-cfntools
 HEAT_TEMPLATES_REPO_DIR=$DEST/heat-templates
 OCC_DIR=$DEST/os-collect-config
-DIB_UTILS_DIR=$DEST/dib-utils
 ORC_DIR=$DEST/os-refresh-config
 OAC_DIR=$DEST/os-apply-config
 
@@ -276,7 +275,6 @@ function install_heat_other {
     git_clone $OAC_REPO $OAC_DIR $OAC_BRANCH
     git_clone $OCC_REPO $OCC_DIR $OCC_BRANCH
     git_clone $ORC_REPO $ORC_DIR $ORC_BRANCH
-    git_clone $DIB_UTILS_REPO $DIB_UTILS_DIR $DIB_UTILS_BRANCH
 }
 
 # start_heat() - Start running processes, including screen
@@ -420,7 +418,7 @@ function create_heat_accounts {
 
 # build_heat_pip_mirror() - Build a pip mirror containing heat agent projects
 function build_heat_pip_mirror {
-    local project_dirs="$OCC_DIR $OAC_DIR $ORC_DIR $HEAT_CFNTOOLS_DIR $DIB_UTILS_DIR"
+    local project_dirs="$OCC_DIR $OAC_DIR $ORC_DIR $HEAT_CFNTOOLS_DIR"
     local projpath proj package
 
     rm -rf $HEAT_PIP_REPO

lib/keystone

@@ -236,7 +236,7 @@ function configure_keystone {
     # Enable caching
     iniset $KEYSTONE_CONF cache enabled "True"
     iniset $KEYSTONE_CONF cache backend "oslo_cache.memcache_pool"
-    iniset $KEYSTONE_CONF cache memcache_servers $SERVICE_HOST:11211
+    iniset $KEYSTONE_CONF cache memcache_servers localhost:11211
 
     # Do not cache the catalog backend due to https://bugs.launchpad.net/keystone/+bug/1537617
     iniset $KEYSTONE_CONF catalog caching "False"

lib/neutron-legacy

@@ -108,7 +108,6 @@ GITDIR["python-neutronclient"]=$DEST/python-neutronclient
 NEUTRON_DIR=$DEST/neutron
 NEUTRON_FWAAS_DIR=$DEST/neutron-fwaas
 NEUTRON_LBAAS_DIR=$DEST/neutron-lbaas
-NEUTRON_VPNAAS_DIR=$DEST/neutron-vpnaas
 NEUTRON_AUTH_CACHE_DIR=${NEUTRON_AUTH_CACHE_DIR:-/var/cache/neutron}
 
 # Support entry points installation of console scripts
@@ -125,9 +124,6 @@ export NEUTRON_TEST_CONFIG_FILE=${NEUTRON_TEST_CONFIG_FILE:-"$NEUTRON_CONF_DIR/d
 # Default provider for load balancer service
 DEFAULT_LB_PROVIDER=LOADBALANCER:Haproxy:neutron_lbaas.services.loadbalancer.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default
 
-# Default provider for VPN service
-DEFAULT_VPN_PROVIDER=VPN:openswan:neutron_vpnaas.services.vpn.service_drivers.ipsec.IPsecVPNDriver:default
-
 # Agent binaries.  Note, binary paths for other agents are set in per-service
 # scripts in lib/neutron_plugins/services/
 AGENT_DHCP_BINARY="$NEUTRON_BIN_DIR/neutron-dhcp-agent"
@@ -139,7 +135,6 @@ AGENT_META_BINARY="$NEUTRON_BIN_DIR/neutron-metadata-agent"
 Q_DHCP_CONF_FILE=$NEUTRON_CONF_DIR/dhcp_agent.ini
 Q_L3_CONF_FILE=$NEUTRON_CONF_DIR/l3_agent.ini
 Q_FWAAS_CONF_FILE=$NEUTRON_CONF_DIR/fwaas_driver.ini
-Q_VPN_CONF_FILE=$NEUTRON_CONF_DIR/vpn_agent.ini
 Q_META_CONF_FILE=$NEUTRON_CONF_DIR/metadata_agent.ini
 
 # Default name for Neutron database
@@ -188,6 +183,7 @@ IPV6_PROVIDER_NETWORK_GATEWAY=${IPV6_PROVIDER_NETWORK_GATEWAY:-}
 # Define the public bridge that will transmit traffic from VMs to the
 # physical network - used by both the OVS and Linux Bridge drivers.
 PUBLIC_BRIDGE=${PUBLIC_BRIDGE:-br-ex}
+PUBLIC_BRIDGE_MTU=${PUBLIC_BRIDGE_MTU:-1500}
 
 # Use flat providernet for public network
 #
@@ -231,10 +227,6 @@ Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True}
 # See _configure_neutron_common() for details about setting it up
 declare -a Q_PLUGIN_EXTRA_CONF_FILES
 
-# List of (optional) config files for VPN device drivers to use with
-# the neutron-q-vpn agent
-declare -a Q_VPN_EXTRA_CONF_FILES
-
 
 Q_RR_CONF_FILE=$NEUTRON_CONF_DIR/rootwrap.conf
 if [[ "$Q_USE_ROOTWRAP" == "False" ]]; then
@@ -359,11 +351,6 @@ source $TOP_DIR/lib/neutron_plugins/services/loadbalancer
 # Hardcoding for 1 service plugin for now
 source $TOP_DIR/lib/neutron_plugins/services/metering
 
-# VPN service plugin functions
-# -------------------------------------------
-# Hardcoding for 1 service plugin for now
-source $TOP_DIR/lib/neutron_plugins/services/vpn
-
 # Firewall Service Plugin functions
 # ---------------------------------
 source $TOP_DIR/lib/neutron_plugins/services/firewall
@@ -375,6 +362,9 @@ else
     Q_USE_SECGROUP=False
 fi
 
+default_route_dev=$(ip route | grep ^default | awk '{print $5}')
+die_if_not_set $LINENO default_route_dev "Failure retrieving default route device"
+
 # Save trace setting
 _XTRACE_NEUTRON=$(set +o | grep xtrace)
 set +o xtrace
@@ -392,19 +382,6 @@ function _determine_config_server {
     echo "$opts"
 }
 
-function _determine_config_vpn {
-    local cfg_file
-    local opts="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE --config-file=$Q_VPN_CONF_FILE"
-    if is_service_enabled q-fwaas; then
-        opts+=" --config-file $Q_FWAAS_CONF_FILE"
-    fi
-    for cfg_file in ${Q_VPN_EXTRA_CONF_FILES[@]}; do
-        opts+=" --config-file $cfg_file"
-    done
-    echo "$opts"
-
-}
-
 function _determine_config_l3 {
     local opts="--config-file $NEUTRON_CONF --config-file=$Q_L3_CONF_FILE"
     if is_service_enabled q-fwaas; then
@@ -419,7 +396,6 @@ function determine_config_files {
     local opts=""
     case "$1" in
         "neutron-server") opts="$(_determine_config_server)" ;;
-        "neutron-vpn-agent") opts="$(_determine_config_vpn)" ;;
         "neutron-l3-agent") opts="$(_determine_config_l3)" ;;
     esac
     if [ -z "$opts" ] ; then
@@ -449,10 +425,6 @@ function configure_neutron {
     if is_service_enabled q-metering; then
         _configure_neutron_metering
     fi
-    if is_service_enabled q-vpn; then
-        deprecated "Configuring q-vpn through devstack is deprecated"
-        _configure_neutron_vpn
-    fi
     if is_service_enabled q-fwaas; then
         deprecated "Configuring q-fwaas through devstack is deprecated"
         _configure_neutron_fwaas
@@ -660,10 +632,6 @@ function install_neutron {
         git_clone $NEUTRON_LBAAS_REPO $NEUTRON_LBAAS_DIR $NEUTRON_LBAAS_BRANCH
         setup_develop $NEUTRON_LBAAS_DIR
     fi
-    if is_service_enabled q-vpn; then
-        git_clone $NEUTRON_VPNAAS_REPO $NEUTRON_VPNAAS_DIR $NEUTRON_VPNAAS_BRANCH
-        setup_develop $NEUTRON_VPNAAS_DIR
-    fi
 
     if [ "$VIRT_DRIVER" == 'xenserver' ]; then
         local dom0_ip
@@ -762,8 +730,6 @@ function start_neutron_other_agents {
 
     if is_service_enabled neutron-vpnaas; then
         :  # Started by plugin
-    elif is_service_enabled q-vpn; then
-        run_process q-vpn "$AGENT_VPN_BINARY $(determine_config_files neutron-vpn-agent)"
     else
         run_process q-l3 "$AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"
     fi
@@ -814,9 +780,6 @@ function stop_neutron_other {
     if is_service_enabled q-fwaas; then
         neutron_fwaas_stop
     fi
-    if is_service_enabled q-vpn; then
-        neutron_vpn_stop
-    fi
     if is_service_enabled q-metering; then
         neutron_metering_stop
     fi
@@ -1070,10 +1033,6 @@ function _configure_neutron_dhcp_agent {
 function _configure_neutron_l3_agent {
     Q_L3_ENABLED=True
 
-    if is_service_enabled q-vpn; then
-        neutron_vpn_configure_agent
-    fi
-
     cp $NEUTRON_DIR/etc/l3_agent.ini.sample $Q_L3_CONF_FILE
 
     iniset $Q_L3_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
@@ -1133,17 +1092,6 @@ function _configure_neutron_fwaas {
     neutron_fwaas_configure_driver
 }
 
-function _configure_neutron_vpn {
-    # Uses oslo config generator to generate VPNaaS sample configuration files
-    (cd $NEUTRON_VPNAAS_DIR && exec ./tools/generate_config_file_samples.sh)
-    if [ -f $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf.sample ]; then
-        cp $NEUTRON_VPNAAS_DIR/etc/neutron_vpnaas.conf.sample $NEUTRON_CONF_DIR/neutron_vpnaas.conf
-        iniset $NEUTRON_CONF_DIR/neutron_vpnaas.conf service_providers service_provider $DEFAULT_VPN_PROVIDER
-    fi
-    neutron_vpn_install_agent_packages
-    neutron_vpn_configure_common
-}
-
 function _configure_dvr {
     iniset $NEUTRON_CONF DEFAULT router_distributed True
     iniset $Q_L3_CONF_FILE DEFAULT agent_mode $Q_DVR_MODE
@@ -1400,6 +1348,11 @@ function _neutron_configure_router_v6 {
 
     # This logic is specific to using the l3-agent for layer 3
     if is_service_enabled q-l3; then
+        # Ensure IPv6 RAs are accepted on the interface with the default route.
+        # This is needed for neutron-based devstack clouds to work in
+        # IPv6-only clouds in the gate. Please do not remove this without
+        # talking to folks in Infra.
+        sudo sysctl -w net.ipv6.conf.$default_route_dev.accept_ra=2
         # Ensure IPv6 forwarding is enabled on the host
         sudo sysctl -w net.ipv6.conf.all.forwarding=1
         # Configure and enable public bridge

lib/neutron_plugins/linuxbridge_agent

@@ -48,6 +48,7 @@ function neutron_plugin_configure_dhcp_agent {
 
 function neutron_plugin_configure_l3_agent {
     sudo brctl addbr $PUBLIC_BRIDGE
+    set_mtu $PUBLIC_BRIDGE $PUBLIC_BRIDGE_MTU
     iniset $Q_L3_CONF_FILE DEFAULT external_network_bridge
     iniset $Q_L3_CONF_FILE DEFAULT l3_agent_manager neutron.agent.l3_agent.L3NATAgentWithStateReport
 }

lib/neutron_plugins/openvswitch_agent

@@ -102,7 +102,7 @@ function neutron_plugin_configure_plugin_agent {
         sudo ovs-vsctl -- --may-exist add-port "br-$GUEST_INTERFACE_DEFAULT" $GUEST_INTERFACE_DEFAULT
 
         # Create external bridge and add port
-        _neutron_ovs_base_add_bridge $PUBLIC_BRIDGE
+        _neutron_ovs_base_add_public_bridge
         sudo ovs-vsctl -- --may-exist add-port $PUBLIC_BRIDGE $PUBLIC_INTERFACE_DEFAULT
 
         # Set bridge mappings to "physnet1:br-$GUEST_INTERFACE_DEFAULT"

lib/neutron_plugins/ovs_base

@@ -105,11 +105,16 @@ function _neutron_ovs_base_configure_l3_agent {
         sudo ip link set $Q_PUBLIC_VETH_EX up
         sudo ip addr flush dev $Q_PUBLIC_VETH_EX
     else
-        _neutron_ovs_base_add_bridge $PUBLIC_BRIDGE
+        _neutron_ovs_base_add_public_bridge
         sudo ovs-vsctl br-set-external-id $PUBLIC_BRIDGE bridge-id $PUBLIC_BRIDGE
     fi
 }
 
+function _neutron_ovs_base_add_public_bridge {
+    _neutron_ovs_base_add_bridge $PUBLIC_BRIDGE
+    set_mtu $PUBLIC_BRIDGE $PUBLIC_BRIDGE_MTU
+}
+
 function _neutron_ovs_base_configure_nova_vif_driver {
     :
 }

lib/neutron_plugins/services/vpn

@@ -1,58 +0,0 @@
-#!/bin/bash
-
-# Neutron VPN plugin
-# ---------------------------
-
-# Save trace setting
-_XTRACE_NEUTRON_VPN=$(set +o | grep xtrace)
-set +o xtrace
-
-
-AGENT_VPN_BINARY="$NEUTRON_BIN_DIR/neutron-vpn-agent"
-VPN_PLUGIN=${VPN_PLUGIN:-"neutron_vpnaas.services.vpn.plugin.VPNDriverPlugin"}
-IPSEC_PACKAGE=${IPSEC_PACKAGE:-"openswan"}
-
-function neutron_vpn_install_agent_packages {
-    install_package $IPSEC_PACKAGE
-    if is_ubuntu && [[ "$IPSEC_PACKAGE" == "strongswan" ]]; then
-        sudo ln -sf /etc/apparmor.d/usr.lib.ipsec.charon /etc/apparmor.d/disable/
-        sudo ln -sf /etc/apparmor.d/usr.lib.ipsec.stroke /etc/apparmor.d/disable/
-        # NOTE: Due to https://bugs.launchpad.net/ubuntu/+source/apparmor/+bug/1387220
-        # one must use 'sudo start apparmor ACTION=reload' for Ubuntu 14.10
-        restart_service apparmor
-    fi
-}
-
-function neutron_vpn_configure_common {
-    _neutron_service_plugin_class_add $VPN_PLUGIN
-    _neutron_deploy_rootwrap_filters $NEUTRON_VPNAAS_DIR
-}
-
-function neutron_vpn_configure_agent {
-    # Uses oslo config generator to generate LBaaS sample configuration files
-    (cd $NEUTRON_VPNAAS_DIR && exec ./tools/generate_config_file_samples.sh)
-    cp $NEUTRON_VPNAAS_DIR/etc/vpn_agent.ini.sample $Q_VPN_CONF_FILE
-    if [[ "$IPSEC_PACKAGE" == "strongswan" ]]; then
-        iniset_multiline $Q_VPN_CONF_FILE vpnagent vpn_device_driver neutron_vpnaas.services.vpn.device_drivers.strongswan_ipsec.StrongSwanDriver
-        if is_fedora; then
-            iniset $Q_VPN_CONF_FILE strongswan default_config_area /usr/share/strongswan/templates/config/strongswan.d
-        fi
-    else
-        iniset_multiline $Q_VPN_CONF_FILE vpnagent vpn_device_driver neutron_vpnaas.services.vpn.device_drivers.ipsec.OpenSwanDriver
-    fi
-}
-
-function neutron_vpn_stop {
-    local ipsec_data_dir=$DATA_DIR/neutron/ipsec
-    local pids
-    if [ -d $ipsec_data_dir ]; then
-        pids=$(find $ipsec_data_dir -name 'pluto.pid' -exec cat {} \;)
-    fi
-    if [ -n "$pids" ]; then
-        sudo kill $pids
-    fi
-    stop_process q-vpn
-}
-
-# Restore xtrace
-$_XTRACE_NEUTRON_VPN

lib/nova_plugins/hypervisor-ironic

@@ -39,7 +39,7 @@ function configure_nova_hypervisor {
     configure_libvirt
     LIBVIRT_FIREWALL_DRIVER=${LIBVIRT_FIREWALL_DRIVER:-"nova.virt.firewall.NoopFirewallDriver"}
 
-    iniset $NOVA_CONF DEFAULT compute_driver nova.virt.ironic.IronicDriver
+    iniset $NOVA_CONF DEFAULT compute_driver ironic.IronicDriver
     iniset $NOVA_CONF DEFAULT firewall_driver $LIBVIRT_FIREWALL_DRIVER
     iniset $NOVA_CONF DEFAULT scheduler_host_manager ironic_host_manager
     iniset $NOVA_CONF DEFAULT ram_allocation_ratio 1.0

lib/tempest

@@ -64,7 +64,11 @@ BUILD_TIMEOUT=${BUILD_TIMEOUT:-196}
 # This must be False on stable branches, as master tempest
 # deps do not match stable branch deps. Set this to True to
 # have tempest installed in DevStack by default.
-INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"}
+INSTALL_TEMPEST=${INSTALL_TEMPEST:-"False"}
+
+# This variable is passed directly to pip install inside the common tox venv
+# that is created
+TEMPEST_PLUGINS=${TEMPEST_PLUGINS:-0}
 
 # Cinder/Volume variables
 TEMPEST_VOLUME_DRIVER=${TEMPEST_VOLUME_DRIVER:-default}
@@ -288,6 +292,7 @@ function configure_tempest {
     fi
     if [ "$VIRT_DRIVER" = "xenserver" ]; then
         iniset $TEMPEST_CONFIG image disk_formats "ami,ari,aki,vhd,raw,iso"
+        iniset $TEMPEST_CONFIG scenario img_disk_format vhd
     fi
 
     # Image Features
@@ -323,11 +328,9 @@ function configure_tempest {
     tmp_cfg_file=$(mktemp)
     cd $TEMPEST_DIR
     if [[ "$OFFLINE" != "True" ]]; then
-        tox -revenv --notest
+        tox -revenv-tempest --notest
     fi
-    # NOTE(mtreinish): Respect constraints on tempest verify-config venv
-    tox -evenv -- pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -r requirements.txt
-    tox -evenv -- tempest verify-config -uro $tmp_cfg_file
+    tox -evenv-tempest -- tempest verify-config -uro $tmp_cfg_file
 
     local compute_api_extensions=${COMPUTE_API_EXTENSIONS:-"all"}
     if [[ ! -z "$DISABLE_COMPUTE_API_EXTENSIONS" ]]; then
@@ -343,7 +346,7 @@ function configure_tempest {
     # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion"
     #       for stable branch on each release which should be changed from "latest" to max supported version of that release.
     local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None}
-    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"}
+    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"2.25"}
     # Reset microversions to None where v2.0 is running which does not support microversion.
     # Both "None" means no microversion testing.
     if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then
@@ -351,16 +354,18 @@ function configure_tempest {
         tempest_compute_max_microversion=None
     fi
     if [ "$tempest_compute_min_microversion" == "None" ]; then
-        inicomment $TEMPEST_CONFIG compute-feature-enabled min_microversion
+        inicomment $TEMPEST_CONFIG compute min_microversion
     else
-        iniset $TEMPEST_CONFIG compute-feature-enabled min_microversion $tempest_compute_min_microversion
+        iniset $TEMPEST_CONFIG compute min_microversion $tempest_compute_min_microversion
     fi
     if [ "$tempest_compute_max_microversion" == "None" ]; then
-        inicomment $TEMPEST_CONFIG compute-feature-enabled max_microversion
+        inicomment $TEMPEST_CONFIG compute max_microversion
     else
-        iniset $TEMPEST_CONFIG compute-feature-enabled max_microversion $tempest_compute_max_microversion
+        iniset $TEMPEST_CONFIG compute max_microversion $tempest_compute_max_microversion
     fi
 
+    # TODO(mriedem): Remove allow_port_security_disabled after liberty-eol.
+    iniset $TEMPEST_CONFIG compute-feature-enabled allow_port_security_disabled True
     iniset $TEMPEST_CONFIG compute-feature-enabled resize True
     iniset $TEMPEST_CONFIG compute-feature-enabled live_migration ${LIVE_MIGRATION_AVAILABLE:-False}
     iniset $TEMPEST_CONFIG compute-feature-enabled change_password False
@@ -393,14 +398,26 @@ function configure_tempest {
 
     # Network
     iniset $TEMPEST_CONFIG network api_version 2.0
-    iniset $TEMPEST_CONFIG network tenant_networks_reachable false
+    iniset $TEMPEST_CONFIG network project_networks_reachable false
     iniset $TEMPEST_CONFIG network public_network_id "$public_network_id"
     iniset $TEMPEST_CONFIG network public_router_id "$public_router_id"
     iniset $TEMPEST_CONFIG network default_network "$FIXED_RANGE"
     iniset $TEMPEST_CONFIG network-feature-enabled ipv6 "$IPV6_ENABLED"
     iniset $TEMPEST_CONFIG network-feature-enabled ipv6_subnet_attributes "$IPV6_SUBNET_ATTRIBUTES_ENABLED"
 
-    local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"}
+    DEFAULT_NET_EXT="address-scope,agent,allowed-address-pairs,binding"
+    DEFAULT_NET_EXT+=",allowed-address-pairs,auto-allocated-topology"
+    DEFAULT_NET_EXT+=",availability_zone,binding,default-subnetpool"
+    DEFAULT_NET_EXT+=",dhcp_agent_scheduler,dns-integration"
+    DEFAULT_NET_EXT+=",dvr,ext-gw-mode,external-net"
+    DEFAULT_NET_EXT+=",extra_dhcp_opt,extraroute,flavors,fwaas"
+    DEFAULT_NET_EXT+=",fwaasrouterinsertion,l3-ha,l3_agent_scheduler,lbaas"
+    DEFAULT_NET_EXT+=",lbaas_agent_scheduler,metering,multi-provider,net-mtu"
+    DEFAULT_NET_EXT+="network-ip-availability,network_availability_zone,port-security"
+    DEFAULT_NET_EXT+=",provider,quotas,rbac-policies,router,router_availability_zone"
+    DEFAULT_NET_EXT+=",security-group,service-type,standard-attr-description"
+    DEFAULT_NET_EXT+=",subnet_allocation,tag,timestamp_core,vpnaas"
+    local network_api_extensions="${NETWORK_API_EXTENSIONS:-$DEFAULT_NET_EXT}"
     if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then
         # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
         network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")}
@@ -442,7 +459,11 @@ function configure_tempest {
     iniset $TEMPEST_CONFIG telemetry-feature-enabled events "True"
 
     # Object Store
-    local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"}
+    DEFAULT_SWIFT_OPT="account_quotas,bulk,bulk_delete,bulk_upload,container_quotas"
+    DEFAULT_SWIFT_OPT+=",container_sync,crossdomain,formpost,ratelimit,slo"
+    DEFAULT_SWIFT_OPT+=",staticweb,tempauth,tempurl"
+
+    local object_storage_api_extensions="${OBJECT_STORAGE_API_EXTENSIONS:-$DEFAULT_SWIFT_OPT}"
     if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then
         # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
         object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")}
@@ -464,7 +485,18 @@ function configure_tempest {
     # TODO(ynesenenko): Remove the volume_services flag when Liberty and Kilo will correct work with host info.
     iniset $TEMPEST_CONFIG volume-feature-enabled volume_services True
 
-    local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"}
+    DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups"
+    DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone"
+    DEFAULT_VOL_EXT+=",os-extended-services,os-extended-snapshot-attributes"
+    DEFAULT_VOL_EXT+=",os-hosts,os-image-create,os-quota-class-sets,os-quota-sets"
+    DEFAULT_VOL_EXT+=",os-services,os-snapshot-actions,os-snapshot-manage,os-snapshot-unmanage"
+    DEFAULT_VOL_EXT+=",os-types-extra-specs"
+    DEFAULT_VOL_EXT+=",os-types-manage,os-used-limits,os-vol-host-attr,os-vol-image-meta"
+    DEFAULT_VOL_EXT+=",os-vol-mig-status-attr,os-vol-tenant-attr,os-volume-actions"
+    DEFAULT_VOL_EXT+=",os-volume-encryption-metadata,os-volume-manage"
+    DEFAULT_VOL_EXT+=",os-volume-replication,os-volume-transfer,os-volume-type-access"
+    DEFAULT_VOL_EXT+=",os-volume-unmanage,qos-specs,scheduler-stats"
+    local volume_api_extensions="${VOLUME_API_EXTENSIONS:-$DEFAULT_VOL_EXT}"
     if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then
         # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
         volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")}
@@ -594,11 +626,16 @@ function install_tempest {
     pip_install tox
     pushd $TEMPEST_DIR
     tox --notest -efull
-    # NOTE(mtreinish) Respect constraints in the tempest full venv, things that
-    # are using a tox job other than full will not be respecting constraints but
-    # running pip install -U on tempest requirements
-    $TEMPEST_DIR/.tox/full/bin/pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -r requirements.txt
-    PROJECT_VENV["tempest"]=${TEMPEST_DIR}/.tox/full
+    PROJECT_VENV["tempest"]=${TEMPEST_DIR}/.tox/tempest
+    popd
+}
+
+# install_tempest_plugins() - Install any specified plugins into the tempest venv
+function install_tempest_plugins {
+    pushd $TEMPEST_DIR
+    if [[ $TEMPEST_PLUGINS != 0 ]] ; then
+        tox -evenv-tempest -- pip install $TEMPEST_PLUGINS
+    fi
     install_tempest_lib
     popd
 }

stack.sh

@@ -799,7 +799,7 @@ if is_service_enabled keystone; then
     fi
 fi
 
-if is_service_enabled s-proxy; then
+if is_service_enabled swift; then
     if is_service_enabled ceilometer; then
         install_ceilometermiddleware
     fi
@@ -1118,7 +1118,7 @@ fi
 # Storage Service
 # ---------------
 
-if is_service_enabled s-proxy; then
+if is_service_enabled swift; then
     echo_summary "Configuring Swift"
     init_swift
 fi
@@ -1172,7 +1172,7 @@ merge_config_group $TOP_DIR/local.conf post-config
 # Only run the services specified in ``ENABLED_SERVICES``
 
 # Launch Swift Services
-if is_service_enabled s-proxy; then
+if is_service_enabled swift; then
     echo_summary "Starting Swift"
     start_swift
 fi

stackrc

@@ -14,9 +14,6 @@ unset LANGUAGE
 LC_ALL=C
 export LC_ALL
 
-# Make tracing more educational
-export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}:   '
-
 # Find the other rc files
 RC_DIR=$(cd $(dirname "${BASH_SOURCE:-$0}") && pwd)
 
@@ -132,6 +129,16 @@ elif [[ -f $RC_DIR/.localrc.auto ]]; then
     source $RC_DIR/.localrc.auto
 fi
 
+# Use color for logging output (only available if syslog is not used)
+LOG_COLOR=$(trueorfalse True LOG_COLOR)
+
+# Make tracing more educational
+if [[ "$LOG_COLOR" == "True" ]]; then
+    export PS4='+\[$(tput setaf 242)\]$(short_source)\[$(tput sgr0)\] '
+else
+    export PS4='+ $(short_source):   '
+fi
+
 # Configure Identity API version: 2.0, 3
 IDENTITY_API_VERSION=${IDENTITY_API_VERSION:-2.0}
 
@@ -205,47 +212,43 @@ REQUIREMENTS_DIR=$DEST/requirements
 
 # block storage service
 CINDER_REPO=${CINDER_REPO:-${GIT_BASE}/openstack/cinder.git}
-CINDER_BRANCH=${CINDER_BRANCH:-master}
+CINDER_BRANCH=${CINDER_BRANCH:-stable/mitaka}
 
 # image catalog service
 GLANCE_REPO=${GLANCE_REPO:-${GIT_BASE}/openstack/glance.git}
-GLANCE_BRANCH=${GLANCE_BRANCH:-master}
+GLANCE_BRANCH=${GLANCE_BRANCH:-stable/mitaka}
 
 # heat service
 HEAT_REPO=${HEAT_REPO:-${GIT_BASE}/openstack/heat.git}
-HEAT_BRANCH=${HEAT_BRANCH:-master}
+HEAT_BRANCH=${HEAT_BRANCH:-stable/mitaka}
 
 # django powered web control panel for openstack
 HORIZON_REPO=${HORIZON_REPO:-${GIT_BASE}/openstack/horizon.git}
-HORIZON_BRANCH=${HORIZON_BRANCH:-master}
+HORIZON_BRANCH=${HORIZON_BRANCH:-stable/mitaka}
 
 # unified auth system (manages accounts/tokens)
 KEYSTONE_REPO=${KEYSTONE_REPO:-${GIT_BASE}/openstack/keystone.git}
-KEYSTONE_BRANCH=${KEYSTONE_BRANCH:-master}
+KEYSTONE_BRANCH=${KEYSTONE_BRANCH:-stable/mitaka}
 
 # neutron service
 NEUTRON_REPO=${NEUTRON_REPO:-${GIT_BASE}/openstack/neutron.git}
-NEUTRON_BRANCH=${NEUTRON_BRANCH:-master}
+NEUTRON_BRANCH=${NEUTRON_BRANCH:-stable/mitaka}
 
 # neutron fwaas service
 NEUTRON_FWAAS_REPO=${NEUTRON_FWAAS_REPO:-${GIT_BASE}/openstack/neutron-fwaas.git}
-NEUTRON_FWAAS_BRANCH=${NEUTRON_FWAAS_BRANCH:-master}
+NEUTRON_FWAAS_BRANCH=${NEUTRON_FWAAS_BRANCH:-stable/mitaka}
 
 # neutron lbaas service
 NEUTRON_LBAAS_REPO=${NEUTRON_LBAAS_REPO:-${GIT_BASE}/openstack/neutron-lbaas.git}
-NEUTRON_LBAAS_BRANCH=${NEUTRON_LBAAS_BRANCH:-master}
-
-# neutron vpnaas service
-NEUTRON_VPNAAS_REPO=${NEUTRON_VPNAAS_REPO:-${GIT_BASE}/openstack/neutron-vpnaas.git}
-NEUTRON_VPNAAS_BRANCH=${NEUTRON_VPNAAS_BRANCH:-master}
+NEUTRON_LBAAS_BRANCH=${NEUTRON_LBAAS_BRANCH:-stable/mitaka}
 
 # compute service
 NOVA_REPO=${NOVA_REPO:-${GIT_BASE}/openstack/nova.git}
-NOVA_BRANCH=${NOVA_BRANCH:-master}
+NOVA_BRANCH=${NOVA_BRANCH:-stable/mitaka}
 
 # object storage service
 SWIFT_REPO=${SWIFT_REPO:-${GIT_BASE}/openstack/swift.git}
-SWIFT_BRANCH=${SWIFT_BRANCH:-master}
+SWIFT_BRANCH=${SWIFT_BRANCH:-stable/mitaka}
 
 ##############
 #
@@ -255,7 +258,7 @@ SWIFT_BRANCH=${SWIFT_BRANCH:-master}
 
 # consolidated openstack requirements
 REQUIREMENTS_REPO=${REQUIREMENTS_REPO:-${GIT_BASE}/openstack/requirements.git}
-REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-master}
+REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-stable/mitaka}
 
 # Tempest test suite
 TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git}
@@ -785,9 +788,6 @@ SYSLOG=$(trueorfalse False SYSLOG)
 SYSLOG_HOST=${SYSLOG_HOST:-$HOST_IP}
 SYSLOG_PORT=${SYSLOG_PORT:-516}
 
-# Use color for logging output (only available if syslog is not used)
-LOG_COLOR=$(trueorfalse True LOG_COLOR)
-
 # Set global ``GIT_DEPTH=<number>`` to limit the history depth of the git clone
 # Set to 0 to disable shallow cloning
 GIT_DEPTH=${GIT_DEPTH:-0}

tests/test_functions.sh

@@ -9,6 +9,22 @@ source $TOP/functions
 
 source $TOP/tests/unittest.sh
 
+echo "Testing generate_hex_string()"
+
+VAL=$(generate_hex_string 16)
+if [[ ${#VAL} -eq 32 ]]; then
+    passed "OK"
+else
+    failed "generate_hex_string 16 failed ${#VAL}"
+fi
+
+VAL=$(generate_hex_string 32)
+if [[ ${#VAL} -eq 64 ]]; then
+    passed "OK"
+else
+    failed "generate_hex_string 32 failed ${#VAL}"
+fi
+
 echo "Testing die_if_not_set()"
 
 bash -c "source $TOP/functions; X=`echo Y && true`; die_if_not_set $LINENO X 'not OK'"

tests/test_localconf.sh

@@ -0,0 +1,475 @@
+#!/usr/bin/env bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# Tests for DevStack INI functions
+
+TOP=$(cd $(dirname "$0")/.. && pwd)
+
+# Import config functions
+source $TOP/inc/ini-config
+
+source $TOP/tests/unittest.sh
+
+echo "Testing INI local.conf functions"
+
+# test that can determine if file has section in specified meta-section
+
+function test_localconf_has_section {
+    local file_localconf
+    local file_conf1
+    local file_conf2
+    file_localconf=`mktemp`
+    file_conf1=`mktemp`
+    file_conf2=`mktemp`
+
+    cat <<- EOF > $file_localconf
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+EOF
+
+    localconf_has_section $file_localconf post-config $file_conf1 conf1_t1
+    assert_equal $? 0
+    localconf_has_section $file_localconf post-config $file_conf1 conf1_t2
+    assert_equal $? 0
+    localconf_has_section $file_localconf post-config $file_conf1 conf1_t3
+    assert_equal $? 0
+    localconf_has_section $file_localconf post-extra $file_conf2 conf2_t1
+    assert_equal $? 0
+    localconf_has_section $file_localconf post-config $file_conf1 conf1_t4
+    assert_equal $? 1
+    localconf_has_section $file_localconf post-install $file_conf1 conf1_t1
+    assert_equal $? 1
+    localconf_has_section $file_localconf local localrc conf1_t2
+    assert_equal $? 1
+    rm -f $file_localconf $file_conf1 $file_conf2
+}
+
+# test that can determine if file has option in specified meta-section and section
+function test_localconf_has_option {
+    local file_localconf
+    local file_conf1
+    local file_conf2
+    file_localconf=`mktemp`
+    file_conf1=`mktemp`
+    file_conf2=`mktemp`
+    cat <<- EOF > $file_localconf
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1 = conf1_t1_val1
+conf1_t1_opt2 = conf1_t1_val2
+conf1_t1_opt3 = conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+EOF
+
+    localconf_has_option $file_localconf local localrc "" LOCALRC_VAR1
+    assert_equal $? 0
+    localconf_has_option $file_localconf local localrc "" LOCALRC_VAR2
+    assert_equal $? 0
+    localconf_has_option $file_localconf local localrc "" LOCALRC_VAR3
+    assert_equal $? 0
+    localconf_has_option $file_localconf post-config $file_conf1 conf1_t1 conf1_t1_opt1
+    assert_equal $? 0
+    localconf_has_option $file_localconf post-config $file_conf1 conf1_t2 conf1_t2_opt2
+    assert_equal $? 0
+    localconf_has_option $file_localconf post-config $file_conf1 conf1_t3 conf1_t3_opt3
+    assert_equal $? 0
+    localconf_has_option $file_localconf post-extra $file_conf2 conf2_t1 conf2_t1_opt2
+    assert_equal $? 0
+    localconf_has_option $file_localconf post-config $file_conf1 conf1_t1_opt4
+    assert_equal $? 1
+    localconf_has_option $file_localconf post-install $file_conf1 conf1_t1_opt1
+    assert_equal $? 1
+    localconf_has_option $file_localconf local localrc conf1_t2 conf1_t2_opt1
+    assert_equal $? 1
+    rm -f $file_localconf $file_conf1 $file_conf2
+}
+
+# test that update option in specified meta-section and section
+function test_localconf_update_option {
+    local file_localconf
+    local file_localconf_expected
+    local file_conf1
+    local file_conf2
+    file_localconf=`mktemp`
+    file_localconf_expected=`mktemp`
+    file_conf1=`mktemp`
+    file_conf2=`mktemp`
+    cat <<- EOF > $file_localconf
+[[local|localrc]]
+LOCALRC_VAR1 = localrc_val1
+LOCALRC_VAR2 = localrc_val2
+LOCALRC_VAR3 = localrc_val3
+
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+EOF
+    cat <<- EOF > $file_localconf_expected
+[[local|localrc]]
+LOCALRC_VAR1 = localrc_val1
+LOCALRC_VAR2 = localrc_val2_update
+LOCALRC_VAR3 = localrc_val3
+
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1_update
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2_update
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3_update
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3_update
+EOF
+
+    localconf_update_option "$SUDO" $file_localconf local localrc "" LOCALRC_VAR2 localrc_val2_update
+    localconf_update_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t1 conf1_t1_opt1 conf1_t1_val1_update
+    localconf_update_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t2 conf1_t2_opt2 conf1_t2_val2_update
+    localconf_update_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t3 conf1_t3_opt3 conf1_t3_val3_update
+    localconf_update_option "$SUDO" $file_localconf post-extra $file_conf2 conf2_t1 conf2_t1_opt3 conf2_t1_val3_update
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    localconf_update_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t2 conf1_t3_opt1 conf1_t3_val1_update
+    localconf_update_option "$SUDO" $file_localconf post-extra $file_conf2 conf2_t1 conf2_t1_opt4 conf2_t1_val4_update
+    localconf_update_option "$SUDO" $file_localconf post-install $file_conf2 conf2_t1 conf2_t1_opt1 conf2_t1_val1_update
+    localconf_update_option "$SUDO" $file_localconf local localrc "" LOCALRC_VAR4 localrc_val4_update
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    rm -f $file_localconf $file_localconf_expected $file_conf1 $file_conf2
+}
+
+# test that add option in specified meta-section and section
+function test_localconf_add_option {
+    local file_localconf
+    local file_localconf_expected
+    local file_conf1
+    local file_conf2
+    file_localconf=`mktemp`
+    file_localconf_expected=`mktemp`
+    file_conf1=`mktemp`
+    file_conf2=`mktemp`
+    cat <<- EOF > $file_localconf
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1 = conf2_t1_val1
+conf2_t1_opt2 = conf2_t1_val2
+conf2_t1_opt3 = conf2_t1_val3
+EOF
+    cat <<- EOF > $file_localconf_expected
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt4 = conf1_t1_val4
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt4 = conf1_t2_val4
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt4 = conf1_t3_val4
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[local|localrc]]
+LOCALRC_VAR4 = localrc_val4
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt4 = conf2_t1_val4
+conf2_t1_opt1 = conf2_t1_val1
+conf2_t1_opt2 = conf2_t1_val2
+conf2_t1_opt3 = conf2_t1_val3
+EOF
+
+    localconf_add_option "$SUDO" $file_localconf local localrc "" LOCALRC_VAR4 localrc_val4
+    localconf_add_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t1 conf1_t1_opt4 conf1_t1_val4
+    localconf_add_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t2 conf1_t2_opt4 conf1_t2_val4
+    localconf_add_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t3 conf1_t3_opt4 conf1_t3_val4
+    localconf_add_option "$SUDO" $file_localconf post-extra $file_conf2 conf2_t1 conf2_t1_opt4 conf2_t1_val4
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    localconf_add_option "$SUDO" $file_localconf local localrc.conf "" LOCALRC_VAR4 localrc_val4_update
+    localconf_add_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t4 conf1_t4_opt1 conf1_t4_val1
+    localconf_add_option "$SUDO" $file_localconf post-extra $file_conf2 conf2_t2 conf2_t2_opt4 conf2_t2_val4
+    localconf_add_option "$SUDO" $file_localconf post-install $file_conf2 conf2_t1 conf2_t1_opt4 conf2_t2_val4
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    rm -f $file_localconf $file_localconf_expected $file_conf1 $file_conf2
+}
+
+# test that add section and option in specified meta-section
+function test_localconf_add_section_and_option {
+    local file_localconf
+    local file_localconf_expected
+    local file_conf1
+    local file_conf2
+    file_localconf=`mktemp`
+    file_localconf_expected=`mktemp`
+    file_conf1=`mktemp`
+    file_conf2=`mktemp`
+    cat <<- EOF > $file_localconf
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+EOF
+    cat <<- EOF > $file_localconf_expected
+[[post-config|$file_conf1]]
+[conf1_t4]
+conf1_t4_opt1 = conf1_t4_val1
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t2]
+conf2_t2_opt1 = conf2_t2_val1
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+EOF
+
+    localconf_add_section_and_option "$SUDO" $file_localconf post-config $file_conf1 conf1_t4 conf1_t4_opt1 conf1_t4_val1
+    localconf_add_section_and_option "$SUDO" $file_localconf post-extra $file_conf2 conf2_t2 conf2_t2_opt1 conf2_t2_val1
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    localconf_add_section_and_option "$SUDO" $file_localconf post-install $file_conf2 conf2_t2 conf2_t2_opt1 conf2_t2_val1
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    rm -f $file_localconf $file_localconf_expected $file_conf1 $file_conf2
+}
+
+# test that add section and option in specified meta-section
+function test_localconf_set {
+    local file_localconf
+    local file_localconf_expected
+    local file_conf1
+    local file_conf2
+    file_localconf=`mktemp`
+    file_localconf_expected=`mktemp`
+    file_conf1=`mktemp`
+    file_conf2=`mktemp`
+    cat <<- EOF > $file_localconf
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2
+LOCALRC_VAR3=localrc_val3
+
+[[post-config|$file_conf1]]
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+EOF
+    cat <<- EOF > $file_localconf_expected
+[[local|localrc]]
+LOCALRC_VAR1=localrc_val1
+LOCALRC_VAR2=localrc_val2_update
+LOCALRC_VAR3=localrc_val3
+
+[[post-config|$file_conf1]]
+[conf1_t4]
+conf1_t4_opt1 = conf1_t4_val1
+[conf1_t1]
+conf1_t1_opt1=conf1_t1_val1
+conf1_t1_opt2=conf1_t1_val2
+conf1_t1_opt3=conf1_t1_val3
+[conf1_t2]
+conf1_t2_opt1=conf1_t2_val1
+conf1_t2_opt2=conf1_t2_val2
+conf1_t2_opt3=conf1_t2_val3
+[conf1_t3]
+conf1_t3_opt1=conf1_t3_val1
+conf1_t3_opt2=conf1_t3_val2
+conf1_t3_opt3=conf1_t3_val3
+
+[[post-extra|$file_conf2]]
+[conf2_t1]
+conf2_t1_opt4 = conf2_t1_val4
+conf2_t1_opt1=conf2_t1_val1
+conf2_t1_opt2=conf2_t1_val2
+conf2_t1_opt3=conf2_t1_val3
+
+[[post-install|/etc/neutron/plugin/ml2/ml2_conf.ini]]
+[ml2]
+ml2_opt1 = ml2_val1
+EOF
+
+    if [[ -n "$SUDO" ]]; then
+        SUDO_ARG="-sudo"
+    else
+        SUDO_ARG=""
+    fi
+    localconf_set $SUDO_ARG $file_localconf post-install /etc/neutron/plugin/ml2/ml2_conf.ini ml2 ml2_opt1 ml2_val1
+    localconf_set $SUDO_ARG $file_localconf local localrc "" LOCALRC_VAR2 localrc_val2_update
+    localconf_set $SUDO_ARG $file_localconf post-config $file_conf1 conf1_t4 conf1_t4_opt1 conf1_t4_val1
+    localconf_set $SUDO_ARG $file_localconf post-extra $file_conf2 conf2_t1 conf2_t1_opt4 conf2_t1_val4
+    result=`cat $file_localconf`
+    result_expected=`cat $file_localconf_expected`
+    assert_equal "$result" "$result_expected"
+    rm -f $file_localconf $file_localconf_expected $file_conf1 $file_conf2
+}
+
+
+test_localconf_has_section
+test_localconf_has_option
+test_localconf_update_option
+test_localconf_add_option
+test_localconf_add_section_and_option
+test_localconf_set

tests/test_refs.sh

@@ -15,7 +15,7 @@
 
 echo "Ensuring we don't have crazy refs"
 
-REFS=`grep BRANCH stackrc | grep -v -- '-master'`
+REFS=`grep BRANCH stackrc | grep -v -- '-master' | grep -v -- '-stable/mitaka'`
 rc=$?
 if [[ $rc -eq 0 ]]; then
     echo "Branch defaults must be master. Found:"

tools/create_userrc.sh

@@ -7,6 +7,22 @@
 # Warning: This script just for development purposes
 
 set -o errexit
+
+# short_source prints out the current location of the caller in a way
+# that strips redundant directories. This is useful for PS4
+# usage. Needed before we start tracing due to how we set
+# PS4. Normally we'd pick this up from stackrc, but that's not sourced
+# here.
+function short_source {
+    saveIFS=$IFS
+    IFS=" "
+    called=($(caller 0))
+    IFS=$saveIFS
+    file=${called[2]}
+    file=${file#$RC_DIR/}
+    printf "%-40s " "$file:${called[1]}:${called[0]}"
+}
+
 set -o xtrace
 
 ACCOUNT_DIR=./accrc

tools/install_pip.sh

@@ -11,7 +11,6 @@
 # - if USE_PYTHON3=True, PYTHON3_VERSION refers to a version already installed
 
 set -o errexit
-set -o xtrace
 
 # Keep track of the current directory
 TOOLS_DIR=$(cd $(dirname "$0") && pwd)
@@ -23,6 +22,9 @@ cd $TOP_DIR
 # Import common functions
 source $TOP_DIR/stackrc
 
+# don't start tracing until after we've sourced the world
+set -o xtrace
+
 FILES=$TOP_DIR/files
 
 PIP_GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
@@ -132,6 +134,9 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then
 fi
 
 set -x
-pip_install -U setuptools
+
+# Note setuptools is part of requirements.txt and we want to make sure
+# we obey any versioning as described there.
+pip_install_gr setuptools
 
 get_versions

tools/worlddump.py

@@ -76,6 +76,31 @@ def _header(name):
     print
 
 
+def _bridge_list():
+    process = subprocess.Popen(['sudo', 'ovs-vsctl', 'list-br'],
+                               stdout=subprocess.PIPE)
+    stdout, _ = process.communicate()
+    return stdout.split()
+
+
+# This method gets a max openflow version supported by openvswitch.
+# For example 'ovs-ofctl --version' displays the following:
+#
+#     ovs-ofctl (Open vSwitch) 2.0.2
+#     Compiled Dec  9 2015 14:08:08
+#     OpenFlow versions 0x1:0x4
+#
+# The above shows that openvswitch supports from OpenFlow10 to OpenFlow13.
+# This method gets max version searching 'OpenFlow versions 0x1:0x'.
+# And return a version value converted to an integer type.
+def _get_ofp_version():
+    process = subprocess.Popen(['ovs-ofctl', '--version'], stdout=subprocess.PIPE)
+    stdout, _ = process.communicate()
+    find_str = 'OpenFlow versions 0x1:0x'
+    offset = stdout.find(find_str)
+    return int(stdout[offset + len(find_str):-1]) - 1
+
+
 def disk_space():
     # the df output
     _header("File System Summary")
@@ -139,15 +164,17 @@ def ovs_dump():
     if not _find_cmd('ovs-vsctl'):
         return
 
-    # NOTE(ihrachys): worlddump is used outside of devstack context (f.e. in
-    # grenade), so there is no single place to determine the bridge names from.
-    # Hardcode for now.
-    bridges = ('br-int', 'br-tun', 'br-ex')
+    bridges = _bridge_list()
+    ofctl_cmds = ('show', 'dump-ports-desc', 'dump-ports', 'dump-flows')
+    ofp_max = _get_ofp_version()
+    vers = 'OpenFlow10'
+    for i in range(1, ofp_max + 1):
+        vers += ',OpenFlow1' + str(i)
     _dump_cmd("sudo ovs-vsctl show")
-    for bridge in bridges:
-        _dump_cmd("sudo ovs-ofctl show %s" % bridge)
-    for bridge in bridges:
-        _dump_cmd("sudo ovs-ofctl dump-flows %s" % bridge)
+    for ofctl_cmd in ofctl_cmds:
+        for bridge in bridges:
+            args = {'vers': vers, 'cmd': ofctl_cmd, 'bridge': bridge}
+            _dump_cmd("sudo ovs-ofctl --protocols=%(vers)s %(cmd)s %(bridge)s" % args)
 
 
 def process_list():