CI: drop nova-cells-v1 job

This job no longer exists, drop it from zuul config to avoid errors. Since some other jobs have unrelated failures, make them non-voting in order to unblock this patch. Change-Id: Ie4cbe2e52adca03fb6064c064e0573ec2b234fd7
Cleanup failing platform job
2023-05-27 12:41:40 +02:00 · 2022-10-09 13:37:42 +02:00 · 2022-08-24 15:16:21 +00:00 · 2022-04-20 12:15:55 +02:00 · 2022-02-11 09:11:50 +00:00 · 2021-10-07 17:59:30 -05:00
36 changed files with 498 additions and 278 deletions
@@ -1,4 +1,5 @@
 [gerrit]
-host=review.openstack.org
+host=review.opendev.org
 port=29418
-project=openstack-dev/devstack.git
+project=openstack/devstack.git
+defaultbranch=stable/stein
@@ -28,36 +28,6 @@
        nodes:
          - controller

- nodeset:
-    name: devstack-single-node-centos-7
-    nodes:
-      - name: controller
-        label: centos-7
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-opensuse-150
-    nodes:
-      - name: controller
-        label: opensuse-150
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-fedora-latest
-    nodes:
-      - name: controller
-        label: fedora-28
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
 - nodeset:
    name: openstack-two-node
    nodes:
@@ -198,10 +168,10 @@
      job.group-vars.peers, which is what is used by multi node jobs for subnode
      nodes (everything but the controller).
    required-projects:
-      - git.openstack.org/openstack-dev/devstack
+      - opendev.org/openstack/devstack
    roles:
-      - zuul: git.openstack.org/openstack-infra/devstack-gate
-      - zuul: git.openstack.org/openstack-infra/openstack-zuul-jobs
+      - zuul: opendev.org/openstack/devstack-gate
+      - zuul: opendev.org/openstack/openstack-zuul-jobs
    vars:
      devstack_localrc:
        DATABASE_PASSWORD: secretdatabase
@@ -239,8 +209,7 @@
        '{{ stage_dir }}/etc': logs
        /var/log/rabbitmq: logs
        /var/log/postgresql: logs
-        /var/log/mysql.err: logs
-        /var/log/mysql.log: logs
+        /var/log/mysql: logs
        /var/log/libvirt: logs
        /etc/sudoers: logs
        /etc/sudoers.d: logs
@@ -307,7 +276,7 @@
      less than the normal minimum set of required-projects.
    nodeset: openstack-single-node-bionic
    required-projects:
-      - git.openstack.org/openstack/requirements
+      - opendev.org/openstack/requirements
    vars:
      devstack_localrc:
        # Multinode specific settings
@@ -365,13 +334,13 @@
      The run playbook consists of a single role, so it can be easily rewritten
      and extended.
    required-projects:
-      - git.openstack.org/openstack/cinder
-      - git.openstack.org/openstack/glance
-      - git.openstack.org/openstack/keystone
-      - git.openstack.org/openstack/neutron
-      - git.openstack.org/openstack/nova
-      - git.openstack.org/openstack/placement
-      - git.openstack.org/openstack/swift
+      - opendev.org/openstack/cinder
+      - opendev.org/openstack/glance
+      - opendev.org/openstack/keystone
+      - opendev.org/openstack/neutron
+      - opendev.org/openstack/nova
+      - opendev.org/openstack/placement
+      - opendev.org/openstack/swift
    timeout: 7200
    vars:
      devstack_localrc:
@@ -460,7 +429,10 @@
          n-cpu: true
          placement-client: true
          # Neutron services
-          neutron-agent: true
+          # We need to keep using the neutron-legacy based services for
+          # now until all issues with the new lib/neutron code are solved
+          q-agt: true
+          # neutron-agent: true
          # Cinder services
          c-bak: true
          c-vol: true
@@ -523,32 +495,15 @@
 # we often have to rush things through devstack to stabilise the gate,
 # and these platforms don't have the round-the-clock support to avoid
 # becoming blockers in that situation.
- job:
-    name: devstack-platform-centos-7
-    parent: tempest-full
-    description: Centos 7 platform test
-    nodeset: devstack-single-node-centos-7
-    voting: false
-
- job:
-    name: devstack-platform-opensuse-150
-    parent: tempest-full
-    description: openSUSE 15.0 platform test
-    nodeset: devstack-single-node-opensuse-150
-    voting: false
-
- job:
-    name: devstack-platform-fedora-latest
-    parent: tempest-full
-    description: Fedora latest platform test
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-
 - job:
    name: devstack-platform-xenial
    parent: tempest-full
    description: Ubuntu Xenial platform test
    nodeset: openstack-single-node-xenial
+    vars:
+      devstack_localrc:
+        # use py3.5 compatible Tempest here.
+        TEMPEST_BRANCH: "23.0.0"
    voting: false

 - job:
@@ -607,6 +562,7 @@

 - job:
    name: devstack-unit-tests
+    nodeset: ubuntu-bionic
    description: |
      Runs unit tests on devstack project.

@@ -621,16 +577,21 @@
      - publish-openstack-docs-pti
    check:
      jobs:
-        - devstack
-        - devstack-xenial
+        - tempest-full:
+            voting: false
+        - tempest-full-py3:
+            voting: false
+        - devstack:
+            voting: false
+        - devstack-xenial:
+            voting: false
        - devstack-ipv6:
            voting: false
-        - devstack-platform-centos-7
-        - devstack-platform-opensuse-150
-        - devstack-platform-fedora-latest
        - devstack-platform-xenial
-        - devstack-multinode
-        - devstack-multinode-xenial
+        - devstack-multinode:
+            voting: false
+        - devstack-multinode-xenial:
+            voting: false
        - devstack-unit-tests
        - openstack-tox-bashate
        - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa:
@@ -641,10 +602,12 @@
              - ^.*\.rst$
              - ^doc/.*$
        - neutron-grenade:
+            voting: false
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
        - neutron-grenade-multinode:
+            voting: false
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@@ -659,21 +622,28 @@
              - ^doc/.*$
    gate:
      jobs:
-        - devstack
-        - devstack-xenial
-        - devstack-multinode
-        - devstack-multinode-xenial
+        # Since these two come from a global template, we need to
+        # live with these running n-v in gate until fixed
+        - tempest-full:
+            voting: false
+        - tempest-full-py3:
+            voting: false
+        # - devstack
+        # - devstack-xenial
+        # - devstack-multinode
+        # - devstack-multinode-xenial
        - devstack-unit-tests
        - openstack-tox-bashate
-        - neutron-grenade-multinode:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        # - neutron-grenade-multinode:
+        #     irrelevant-files:
+        #       - ^.*\.rst$
+        #       - ^doc/.*$
        - neutron-tempest-linuxbridge:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
        - neutron-grenade:
+            voting: false
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@@ -698,10 +668,6 @@

    experimental:
      jobs:
-        - nova-cells-v1:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - nova-next
        - neutron-fullstack-with-uwsgi
        - neutron-functional-with-uwsgi
@@ -10,6 +10,7 @@ Walk through various setups used by stackers

 .. toctree::
   :glob:
+   :hidden:
   :maxdepth: 1

   guides/single-vm
@@ -68,6 +69,11 @@ Nova and devstack

 Guide to working with nova features :doc:`Nova and devstack <guides/nova>`.

+Configure Load-Balancer Version 2
+-----------------------------------
+
+Guide on :doc:`Configure Load-Balancer Version 2 <guides/devstack-with-lbaas-v2>`.
+
 Deploying DevStack with LDAP
 ----------------------------

@@ -589,6 +589,18 @@ function git_clone {
        fi
    fi

+    # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
+    # about how we clone and work with repos.  Mark them safe globally
+    # as a work-around.
+    #
+    # NOTE(danms): On bionic (and likely others) git-config may write
+    # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+    # writes these changes to /etc/gitconfig which is more
+    # discoverable anyway.
+    #
+    # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
+    sudo git config --system --add safe.directory ${git_dest}
+
    # print out the results so we know what change was used in the logs
    cd $git_dest
    git show --oneline | head -1
@@ -49,16 +49,9 @@ function get_python_exec_prefix {
    fi
    $xtrace

-    if python3_enabled && [[ "$os_VENDOR" == "CentOS" ]] || \
-    [[ "$os_VENDOR" == "Fedora" && $os_RELEASE -gt 26 ]]; then
-        # Default Python 3 install prefix changed to /usr/local in Fedora 27:
-        # https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe
-        echo "/usr/local/bin"
-    elif is_fedora || is_suse; then
-        echo "/usr/bin"
-    else
-        echo "/usr/local/bin"
-    fi
+    local PYTHON_PATH=/usr/local/bin
+    ( is_fedora && ! python3_enabled ) || is_suse && PYTHON_PATH=/usr/bin
+    echo $PYTHON_PATH
 }

 # Wrapper for ``pip install`` that only installs versions of libraries
@@ -88,34 +81,6 @@ function pip_install_gr_extras {
    pip_install $clean_name[$extras]
 }

-# Determine the python versions supported by a package
-function get_python_versions_for_package {
-    local name=$1
-    cd $name && python setup.py --classifiers \
-        | grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
-}
-
-# Check for python3 classifier in local directory
-function check_python3_support_for_package_local {
-    local name=$1
-    cd $name
-    set +e
-    classifier=$(python setup.py --classifiers \
-        | grep 'Programming Language :: Python :: 3')
-    set -e
-    echo $classifier
-}
-
-# Check for python3 classifier on pypi
-function check_python3_support_for_package_remote {
-    local name=$1
-    set +e
-    classifier=$(curl -s -L "https://pypi.python.org/pypi/$name/json" \
-        | grep '"Programming Language :: Python :: 3"')
-    set -e
-    echo $classifier
-}
-
 # python3_enabled_for() assumes the service(s) specified as arguments are
 # enabled for python 3 unless explicitly disabled. See python3_disabled_for().
 #
@@ -266,52 +231,26 @@ function pip_install {
            cmd_pip=$(get_pip_command $PYTHON2_VERSION)
            local sudo_pip="sudo -H"
            if python3_enabled; then
-                # Look at the package classifiers to find the python
-                # versions supported, and if we find the version of
-                # python3 we've been told to use, use that instead of the
-                # default pip
-                local python_versions
-
                # Special case some services that have experimental
                # support for python3 in progress, but don't claim support
                # in their classifier
                echo "Check python version for : $package_dir"
                if python3_disabled_for ${package_dir##*/}; then
                    echo "Explicitly using $PYTHON2_VERSION version to install $package_dir based on DISABLED_PYTHON3_PACKAGES"
-                elif python3_enabled_for ${package_dir##*/}; then
-                    echo "Using $PYTHON3_VERSION version to install $package_dir based on default behavior"
-                    sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
-                    cmd_pip=$(get_pip_command $PYTHON3_VERSION)
-                elif [[ -d "$package_dir" ]]; then
-                    python_versions=$(get_python_versions_for_package $package_dir)
-                    if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
-                        echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on classifiers"
-                        sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
-                        cmd_pip=$(get_pip_command $PYTHON3_VERSION)
-                    else
-                        # The package may not have yet advertised python3.5
-                        # support so check for just python3 classifier and log
-                        # a warning.
-                        python3_classifier=$(check_python3_support_for_package_local $package_dir)
-                        if [[ ! -z "$python3_classifier" ]]; then
-                            echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on local package settings"
-                            sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
-                            cmd_pip=$(get_pip_command $PYTHON3_VERSION)
-                        else
-                            echo "WARNING: Did not find python 3 classifier for local package $package_dir"
-                        fi
-                    fi
                else
-                    # Check pypi as we don't have the package on disk
-                    package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*')
-                    python3_classifier=$(check_python3_support_for_package_remote $package)
-                    if [[ ! -z "$python3_classifier" ]]; then
-                        echo "Automatically using $PYTHON3_VERSION version to install $package based on remote package settings"
-                        sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
-                        cmd_pip=$(get_pip_command $PYTHON3_VERSION)
-                    else
-                        echo "WARNING: Did not find python 3 classifier for remote package $package_dir"
-                    fi
+                    # For everything that is not explicitly blacklisted with
+                    # DISABLED_PYTHON3_PACKAGES, assume it supports python3
+                    # and we will let pip sort out the install, regardless of
+                    # the package being local or remote.
+                    echo "Using $PYTHON3_VERSION version to install $package_dir based on default behavior"
+                    # See
+                    #  https://github.com/pypa/setuptools/issues/2232
+                    #  http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
+                    # this makes setuptools >=50 use the platform distutils.
+                    # We only want to do this on global pip installs, not if
+                    # installing in a virtualenv
+                    sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
+                    cmd_pip=$(get_pip_command $PYTHON3_VERSION)
                fi
            fi
        fi
@@ -333,13 +272,6 @@ function pip_install {

    $xtrace

-    # Also install test requirements
-    local install_test_reqs=""
-    local test_req="${package_dir}/test-requirements.txt"
-    if [[ -e "$test_req" ]]; then
-        install_test_reqs="-r $test_req"
-    fi
-
    # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
    # the same behaviour of setuptools before version 25.0.0.
    # related issue: https://github.com/pypa/pip/issues/3874
@@ -349,7 +281,7 @@ function pip_install {
        no_proxy="${no_proxy:-}" \
        PIP_FIND_LINKS=$PIP_FIND_LINKS \
        SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
-        $cmd_pip $upgrade $install_test_reqs \
+        $cmd_pip $upgrade \
        $@
    result=$?

@@ -82,26 +82,52 @@ function install_apache_uwsgi {
        apxs="apxs"
    fi

-    # Ubuntu xenial is back level on uwsgi so the proxy doesn't
-    # actually work. Hence we have to build from source for now.
+    # This varies based on packaged/installed.  If we've
+    # pip_installed, then the pip setup will only build a "python"
+    # module that will be either python2 or python3 depending on what
+    # it was built with.
    #
-    # Centos 7 actually has the module in epel, but there was a big
-    # push to disable epel by default. As such, compile from source
-    # there as well.
+    # For package installs, the distro ships both plugins and you need
+    # to select the right one ... it will not be autodetected.
+    if python3_enabled; then
+        UWSGI_PYTHON_PLUGIN=python3
+    else
+        UWSGI_PYTHON_PLUGIN=python
+    fi

-    local dir
-    dir=$(mktemp -d)
-    pushd $dir
-    pip_install uwsgi
-    pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
-    local uwsgi
-    uwsgi=$(ls uwsgi*)
-    tar xvf $uwsgi
-    cd uwsgi*/apache2
-    sudo $apxs -i -c mod_proxy_uwsgi.c
-    popd
-    # delete the temp directory
-    sudo rm -rf $dir
+    if is_ubuntu; then
+        install_package uwsgi \
+                        uwsgi-plugin-python \
+                        uwsgi-plugin-python3 \
+                        libapache2-mod-proxy-uwsgi
+    elif [[ $os_VENDOR == "Fedora" ]]; then
+        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
+        # default; the mod_proxy_uwsgi package actually conflicts now.
+        # See:
+        #  https://bugzilla.redhat.com/show_bug.cgi?id=1574335
+        #
+        # Thus there is nothing else to do after this install
+        install_package uwsgi \
+                        uwsgi-plugin-python3
+    else
+        # Centos actually has the module in epel, but there was a big
+        # push to disable epel by default. As such, compile from source
+        # there.
+        local dir
+        dir=$(mktemp -d)
+        pushd $dir
+        pip_install uwsgi
+        pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
+        local uwsgi
+        uwsgi=$(ls uwsgi*)
+        tar xvf $uwsgi
+        cd uwsgi*/apache2
+        sudo $apxs -i -c mod_proxy_uwsgi.c
+        popd
+        # delete the temp directory
+        sudo rm -rf $dir
+        UWSGI_PYTHON_PLUGIN=python
+    fi

    if is_ubuntu || is_suse ; then
        # we've got to enable proxy and proxy_uwsgi for this to work
@@ -265,7 +291,7 @@ function write_uwsgi_config {
    # configured after graceful shutdown
    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
    # uwsgi recommends this to prevent thundering herd on accept.
    iniset "$file" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
@@ -318,7 +344,7 @@ function write_local_uwsgi_http_config {
    iniset "$file" uwsgi die-on-term true
    iniset "$file" uwsgi exit-on-reload false
    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
    # uwsgi recommends this to prevent thundering herd on accept.
    iniset "$file" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
@@ -507,7 +507,7 @@ function start_cinder {
                start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT
            fi
        else
-            run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF"
+            run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF"
            cinder_url=$service_protocol://$SERVICE_HOST/volume/v3
        fi
    fi
@@ -343,7 +343,7 @@ function start_glance {

    run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf"
    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
-        run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
+        run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
    else
        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR"
    fi
@@ -29,7 +29,7 @@ GITDIR["pbr"]=$DEST/pbr
 # install_infra() - Collect source and prepare
 function install_infra {
    local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv"
-    [ ! -d $PIP_VIRTUAL_ENV ] && virtualenv $PIP_VIRTUAL_ENV
+    [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV
    # We don't care about testing git pbr in the requirements venv.
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR
@@ -131,6 +131,9 @@ KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT=${KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT:-2}
 # however may not be suitable for real production.
 KEYSTONE_PASSWORD_HASH_ROUNDS=${KEYSTONE_PASSWORD_HASH_ROUNDS:-4}

+# Cache settings
+KEYSTONE_ENABLE_CACHE=${KEYSTONE_ENABLE_CACHE:-True}
+
 # Functions
 # ---------

@@ -213,9 +216,9 @@ function configure_keystone {
    iniset $KEYSTONE_CONF resource driver "$KEYSTONE_RESOURCE_BACKEND"

    # Enable caching
-    iniset $KEYSTONE_CONF cache enabled "True"
-    iniset $KEYSTONE_CONF cache backend "dogpile.cache.memcached"
-    iniset $KEYSTONE_CONF cache memcache_servers localhost:11211
+    iniset $KEYSTONE_CONF cache enabled $KEYSTONE_ENABLE_CACHE
+    iniset $KEYSTONE_CONF cache backend $CACHE_BACKEND
+    iniset $KEYSTONE_CONF cache memcache_servers $MEMCACHE_SERVERS

    iniset_rpc_backend keystone $KEYSTONE_CONF oslo_messaging_notifications

@@ -515,7 +518,7 @@ function start_keystone {
        enable_apache_site keystone
        restart_apache_server
    else # uwsgi
-        run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" ""
+        run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" ""
    fi

    echo "Waiting for keystone to start..."
@@ -124,13 +124,15 @@ function init_lvm_volume_group {
    local vg=$1
    local size=$2

-    # Start the lvmetad and tgtd services
-    if is_fedora || is_suse; then
-        # services is not started by default
+    # Start the lvmetad service
+    if is_fedora ; then
+        # service is not started by default
        start_service lvm2-lvmetad
-        if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
-            start_service tgtd
-        fi
+    fi
+
+    # Start the tgtd service on Fedora and SUSE if tgtadm is used
+    if  is_fedora || is_suse  && [[ "$CINDER_ISCSI_HELPER" = "tgtadm" ]]; then
+        start_service tgtd
    fi

    # Start with a clean volume group
@@ -117,7 +117,9 @@ function is_neutron_enabled {
 # Test if any Neutron services are enabled
 # is_neutron_enabled
 function is_neutron_legacy_enabled {
-    [[ ,${DISABLED_SERVICES} =~ ,"neutron" ]] && return 1
+    # first we need to remove all "neutron-" from DISABLED_SERVICES list
+    disabled_services_copy=$(echo $DISABLED_SERVICES | sed 's/neutron-//g')
+    [[ ,${disabled_services_copy} =~ ,"neutron" ]] && return 1
    [[ ,${ENABLED_SERVICES} =~ ,"q-" ]] && return 0
    return 1
 }
@@ -460,7 +462,7 @@ function start_neutron_api {
    done

    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
-        run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
+        run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
        neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST/networking/
        enable_service neutron-rpc-server
        run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $opts"
@@ -486,7 +486,7 @@ function start_neutron_service_and_check {
    # Start the Neutron service
    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
        enable_service neutron-api
-        run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
+        run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
        neutron_url=$Q_PROTOCOL://$Q_HOST/networking/
        enable_service neutron-rpc-server
        run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options"
@@ -395,8 +395,6 @@ function _neutron_configure_router_v6 {

    # This logic is specific to using the l3-agent for layer 3
    if is_service_enabled q-l3 || is_service_enabled neutron-l3; then
-        # Ensure IPv6 forwarding is enabled on the host
-        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        # if the Linux host considers itself to be a router then it will
        # ignore all router advertisements
        # Ensure IPv6 RAs are accepted on interfaces with a default route.
@@ -409,6 +407,8 @@ function _neutron_configure_router_v6 {
            # device name would be reinterpreted as a slash, causing an error.
            sudo sysctl -w net/ipv6/conf/$d/accept_ra=2
        done
+        # Ensure IPv6 forwarding is enabled on the host
+        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        # Configure and enable public bridge
        # Override global IPV6_ROUTER_GW_IP with the true value from neutron
        IPV6_ROUTER_GW_IP=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" port list -c 'Fixed IP Addresses' | grep $ipv6_pub_subnet_id | awk -F'ip_address' '{ print $2 }' | cut -f2 -d\' | tr '\n' ' ')
@@ -94,6 +94,7 @@ NOVA_SERVICE_PROTOCOL=${NOVA_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 NOVA_SERVICE_LOCAL_HOST=${NOVA_SERVICE_LOCAL_HOST:-$SERVICE_LOCAL_HOST}
 NOVA_SERVICE_LISTEN_ADDRESS=${NOVA_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)}
 METADATA_SERVICE_PORT=${METADATA_SERVICE_PORT:-8775}
+NOVA_ENABLE_CACHE=${NOVA_ENABLE_CACHE:-True}

 # Option to enable/disable config drive
 # NOTE: Set ``FORCE_CONFIG_DRIVE="False"`` to turn OFF config drive
@@ -183,6 +184,15 @@ TEST_FLOATING_RANGE=${TEST_FLOATING_RANGE:-192.168.253.0/29}
 # and Glance.
 NOVA_USE_SERVICE_TOKEN=$(trueorfalse False NOVA_USE_SERVICE_TOKEN)

+# Format for notifications. Nova defaults to "both".
+# Other options include "versioned" and "unversioned".
+NOVA_NOTIFICATION_FORMAT=${NOVA_NOTIFICATION_FORMAT:-both}
+
+# Timeout for servers to gracefully shutdown the OS during operations
+# like shelve, rescue, stop, rebuild. Defaults to 0 since the default
+# image in devstack is CirrOS.
+NOVA_SHUTDOWN_TIMEOUT=${NOVA_SHUTDOWN_TIMEOUT:-0}
+
 # Functions
 # ---------

@@ -424,6 +434,7 @@ function create_nova_conf {
    iniset $NOVA_CONF DEFAULT instance_name_template "${INSTANCE_NAME_PREFIX}%08x"
    iniset $NOVA_CONF DEFAULT osapi_compute_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
    iniset $NOVA_CONF DEFAULT metadata_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
+    iniset $NOVA_CONF DEFAULT shutdown_timeout $NOVA_SHUTDOWN_TIMEOUT

    iniset $NOVA_CONF key_manager backend nova.keymgr.conf_key_mgr.ConfKeyManager

@@ -433,8 +444,8 @@ function create_nova_conf {
        iniset $NOVA_CONF DEFAULT bindir "/usr/bin"
    fi

-    # only setup database connections if there are services that
-    # require them running on the host. The ensures that n-cpu doesn't
+    # only setup database connections and cache backend if there are services
+    # that require them running on the host. The ensures that n-cpu doesn't
    # leak a need to use the db in a multinode scenario.
    if is_service_enabled n-api n-cond n-sched; then
        # If we're in multi-tier cells mode, we want our control services pointing
@@ -451,6 +462,13 @@ function create_nova_conf {

        iniset $NOVA_CONF database connection `database_connection_url $db`
        iniset $NOVA_CONF api_database connection `database_connection_url nova_api`
+
+        # Cache related settings
+        # Those settings aren't really needed in n-cpu thus it is configured
+        # only on nodes which runs controller services
+        iniset $NOVA_CONF cache enabled $NOVA_ENABLE_CACHE
+        iniset $NOVA_CONF cache backend $CACHE_BACKEND
+        iniset $NOVA_CONF cache memcache_servers $MEMCACHE_SERVERS
    fi

    if is_service_enabled n-api; then
@@ -517,6 +535,7 @@ function create_nova_conf {
    # enable notifications, but it will allow them to function when enabled.
    iniset $NOVA_CONF oslo_messaging_notifications driver "messagingv2"
    iniset $NOVA_CONF oslo_messaging_notifications transport_url $(get_notification_url)
+    iniset $NOVA_CONF notifications notification_format "$NOVA_NOTIFICATION_FORMAT"
    iniset_rpc_backend nova $NOVA_CONF

    iniset $NOVA_CONF DEFAULT osapi_compute_workers "$API_WORKERS"
@@ -885,7 +904,7 @@ function start_nova_api {
            start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT
        fi
    else
-        run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF"
+        run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF"
        nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/
    fi

@@ -1001,7 +1020,7 @@ function start_nova_rest {
    if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then
        run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf"
    else
-        run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF"
+        run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF"
    fi

    # nova-consoleauth always runs globally
@@ -154,7 +154,7 @@ function install_placement {
 # start_placement_api() - Start the API processes ahead of other things
 function start_placement_api {
    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
-        run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF"
+        run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF"
    else
        enable_apache_site placement-api
        restart_apache_server
@@ -757,7 +757,9 @@ function init_swift {

 function install_swift {
    git_clone $SWIFT_REPO $SWIFT_DIR $SWIFT_BRANCH
-    setup_develop $SWIFT_DIR
+    # keystonemiddleware needs to be installed via keystone extras as defined
+    # in setup.cfg, see bug #1909018 for more details.
+    setup_develop $SWIFT_DIR keystone
    if [ "$SWIFT_USE_MOD_WSGI" == "True" ]; then
        install_apache_wsgi
    fi
@@ -57,7 +57,7 @@ BUILD_TIMEOUT=${BUILD_TIMEOUT:-196}
 # This must be False on stable branches, as master tempest
 # deps do not match stable branch deps. Set this to True to
 # have tempest installed in DevStack by default.
-INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"}
+INSTALL_TEMPEST=${INSTALL_TEMPEST:-"False"}

 # This variable is passed directly to pip install inside the common tox venv
 # that is created
@@ -110,6 +110,21 @@ function image_size_in_gib {
    echo $size | python -c "import math; print int(math.ceil(float(int(raw_input()) / 1024.0 ** 3)))"
 }

+function set_tempest_venv_constraints {
+    local tmp_c
+    tmp_c=$1
+    if [[ $TEMPEST_VENV_UPPER_CONSTRAINTS == "master" ]]; then
+        (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > $tmp_c
+    else
+        echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env."
+        cat $TEMPEST_VENV_UPPER_CONSTRAINTS > $tmp_c
+        # NOTE: setting both tox env var and once Tempest start using new var
+        # TOX_CONSTRAINTS_FILE then we can remove the old one.
+        export UPPER_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS
+        export TOX_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS
+    fi
+}
+
 # configure_tempest() - Set config files, create data dirs, etc
 function configure_tempest {
    if [[ "$INSTALL_TEMPEST" == "True" ]]; then
@@ -201,13 +216,13 @@ function configure_tempest {
            if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then
                # Determine the flavor disk size based on the image size.
                disk=$(image_size_in_gib $image_uuid)
-                openstack flavor create --id 42 --ram 64 --disk $disk --vcpus 1 m1.nano
+                openstack flavor create --id 42 --ram 128 --disk $disk --vcpus 1 m1.nano
            fi
            flavor_ref=42
            if [[ ! ( $available_flavors =~ 'm1.micro' ) ]]; then
                # Determine the alt flavor disk size based on the alt image size.
                disk=$(image_size_in_gib $image_uuid_alt)
-                openstack flavor create --id 84 --ram 128 --disk $disk --vcpus 1 m1.micro
+                openstack flavor create --id 84 --ram 192 --disk $disk --vcpus 1 m1.micro
            fi
            flavor_ref_alt=84
        else
@@ -358,7 +373,7 @@ function configure_tempest {
    # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion"
    #       for stable branch on each release which should be changed from "latest" to max supported version of that release.
    local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None}
-    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"}
+    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"2.72"}
    # Reset microversions to None where v2.0 is running which does not support microversion.
    # Both "None" means no microversion testing.
    if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then
@@ -383,6 +398,7 @@ function configure_tempest {
    iniset $TEMPEST_CONFIG compute-feature-enabled block_migration_for_live_migration ${USE_BLOCK_MIGRATION_FOR_LIVE_MIGRATION:-False}
    iniset $TEMPEST_CONFIG compute-feature-enabled live_migrate_back_and_forth ${LIVE_MIGRATE_BACK_AND_FORTH:-False}
    iniset $TEMPEST_CONFIG compute-feature-enabled attach_encrypted_volume ${ATTACH_ENCRYPTED_VOLUME_AVAILABLE:-True}
+    iniset $TEMPEST_CONFIG compute-feature-enabled xenapi_apis True
    if is_service_enabled n-cell; then
        # Cells doesn't support shelving/unshelving
        iniset $TEMPEST_CONFIG compute-feature-enabled shelve False
@@ -467,8 +483,13 @@ function configure_tempest {
        TEMPEST_EXTEND_ATTACHED_VOLUME=${TEMPEST_EXTEND_ATTACHED_VOLUME:-True}
    fi
    iniset $TEMPEST_CONFIG volume-feature-enabled extend_attached_volume $(trueorfalse False TEMPEST_EXTEND_ATTACHED_VOLUME)
+    # Only turn on TEMPEST_VOLUME_REVERT_TO_SNAPSHOT by default for "lvm" backends
+    if [[ "$CINDER_ENABLED_BACKENDS" == *"lvm"* ]]; then
+        TEMPEST_VOLUME_REVERT_TO_SNAPSHOT=${TEMPEST_VOLUME_REVERT_TO_SNAPSHOT:-True}
+    fi
+    iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT)
    local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None}
-    local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"}
+    local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"3.59"}
    # Reset microversions to None where v2 is running which does not support microversion.
    # Both "None" means no microversion testing.
    if [[ "$TEMPEST_VOLUME_TYPE" == "volumev2" ]]; then
@@ -584,17 +605,19 @@ function configure_tempest {
        tox -revenv-tempest --notest
    fi

-    # The requirements might be on a different branch, while tempest needs master requirements.
-    (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > u-c-m.txt
-    tox -evenv-tempest -- pip install -c u-c-m.txt -r requirements.txt
+    local tmp_u_c_m
+    tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
+    set_tempest_venv_constraints $tmp_u_c_m
+    tox -evenv-tempest -- pip install -c $tmp_u_c_m -r requirements.txt
+    rm -f $tmp_u_c_m

    # Auth:
    iniset $TEMPEST_CONFIG auth tempest_roles "member"
    if [[ $TEMPEST_USE_TEST_ACCOUNTS == "True" ]]; then
        if [[ $TEMPEST_HAS_ADMIN == "True" ]]; then
-            tox -evenv-tempest -- tempest-account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-tenant-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml
+            tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY --with-admin etc/accounts.yaml
        else
-            tox -evenv-tempest -- tempest-account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-tenant-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml
+            tox -evenv-tempest -- tempest account-generator -c $TEMPEST_CONFIG --os-username $admin_username --os-password "$password" --os-project-name $admin_project_name -r $TEMPEST_CONCURRENCY etc/accounts.yaml
        fi
        iniset $TEMPEST_CONFIG auth use_dynamic_credentials False
        iniset $TEMPEST_CONFIG auth test_accounts_file "etc/accounts.yaml"
@@ -622,16 +645,35 @@ function configure_tempest {
        DISABLE_NETWORK_API_EXTENSIONS+=", l3_agent_scheduler"
    fi

-    local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"}
+    DEFAULT_NET_EXT="address-scope,agent,allowed-address-pairs,auto-allocated-topology"
+    DEFAULT_NET_EXT+=",availability_zone,binding,default-subnetpools,dhcp_agent_scheduler"
+    DEFAULT_NET_EXT+=",dvr,ext-gw-mode,external-net,extra_dhcp_opt,extraroute,flavors"
+    DEFAULT_NET_EXT+=",l3-flavors,l3-ha,l3_agent_scheduler,multi-provider,net-mtu"
+    DEFAULT_NET_EXT+=",network-ip-availability,network_availability_zone,pagination"
+    DEFAULT_NET_EXT+=",port-security,project-id,provider,quotas,quota_details,rbac-policies"
+    DEFAULT_NET_EXT+=",revision-if-match,router,router_availability_zone,security-group,service-type,sorting"
+    DEFAULT_NET_EXT+=",standard-attr-description,standard-attr-revisions,standard-attr-tag,standard-attr-timestamp"
+    DEFAULT_NET_EXT+=",subnet-service-types,subnet_allocation,net-mtu-writable,ip-substring-filtering"
+    DEFAULT_NET_EXT+=",availability_zone_filter,filter-validation,empty-string-filtering,port-mac-address-regenerate"
+    DEFAULT_NET_EXT+=",port-security-groups-filtering,fip-port-details,binding-extended"
+    DEFAULT_NET_EXT+=",subnet_onboard,l3-port-ip-change-not-allowed,agent-resources-synced"
+    DEFAULT_NET_EXT+=",floatingip-pools,rbac-security-groups"
+    local network_api_extensions=${NETWORK_API_EXTENSIONS:-$DEFAULT_NET_EXT}
    if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then
        # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
        network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")}
        # Remove disabled extensions
        network_api_extensions=$(remove_disabled_extensions $network_api_extensions $DISABLE_NETWORK_API_EXTENSIONS)
    fi
+    if [[ -n "$ADDITIONAL_NETWORK_API_EXTENSIONS" ]] && [[ "$network_api_extensions" != "all" ]]; then
+        network_api_extensions+=",$ADDITIONAL_NETWORK_API_EXTENSIONS"
+    fi
    iniset $TEMPEST_CONFIG network-feature-enabled api_extensions $network_api_extensions
    # Swift API Extensions
-    local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"}
+    DEFAULT_SWIFT_OPT="account_quotas,bulk_delete,bulk_upload,container_quotas"
+    DEFAULT_SWIFT_OPT+=",container_sync,crossdomain,formpost,ratelimit,slo"
+    DEFAULT_SWIFT_OPT+=",staticweb,tempauth,tempurl,versioned_writes"
+    local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$DEFAULT_SWIFT_OPT}
    if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then
        # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
        object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")}
@@ -639,8 +681,30 @@ function configure_tempest {
        object_storage_api_extensions=$(remove_disabled_extensions $object_storage_api_extensions $DISABLE_STORAGE_API_EXTENSIONS)
    fi
    iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions
+
+    # This feature list for Neutron used from Victoria, the aim is to skip tests
+    # for features in Neutron that has no API extension, or can't be detected
+    # otherwise from the API.
+    # It is added here to avoid stable gating to run these tests (like for IPv6 metadata
+    # in neutron-tempest-plugin or updating min_bw QoS values for ports already attached
+    # to VMs).
+    # To check new features grep neutron-tempest-plugin and tempest for
+    # is_network_feature_enabled, and please check with Neutron team.
+    local network_available_features=""
+    iniset $TEMPEST_CONFIG network-feature-enabled available_features $network_available_features
    # Cinder API Extensions
-    local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"}
+    DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups"
+    DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone"
+    DEFAULT_VOL_EXT+=",os-extended-services,os-extended-snapshot-attributes"
+    DEFAULT_VOL_EXT+=",os-hosts,os-image-create,os-quota-class-sets,os-quota-sets"
+    DEFAULT_VOL_EXT+=",os-services,os-snapshot-actions,os-snapshot-manage"
+    DEFAULT_VOL_EXT+=",os-snapshot-unmanage,os-types-extra-specs,os-types-manage"
+    DEFAULT_VOL_EXT+=",os-used-limits,os-vol-host-attr,os-vol-image-meta"
+    DEFAULT_VOL_EXT+=",os-vol-mig-status-attr,os-vol-tenant-attr,os-volume-actions"
+    DEFAULT_VOL_EXT+=",os-volume-encryption-metadata,os-volume-manage"
+    DEFAULT_VOL_EXT+=",os-volume-transfer,os-volume-type-access"
+    DEFAULT_VOL_EXT+=",os-volume-unmanage,qos-specs,scheduler-stats"
+    local volume_api_extensions=${VOLUME_API_EXTENSIONS:-$DEFAULT_VOL_EXT}
    if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then
        # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
        volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")}
@@ -658,12 +722,22 @@ function install_tempest {
    git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH
    pip_install 'tox!=2.8.0'
    pushd $TEMPEST_DIR
+    # NOTE(gmann): checkout the TEMPEST_BRANCH in case TEMPEST_BRANCH
+    # is tag name not master. git_clone would not checkout tag because
+    # TEMPEST_DIR already exist until RECLONE is true.
+    git checkout $TEMPEST_BRANCH
+
+    local tmp_u_c_m
+    tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
+    set_tempest_venv_constraints $tmp_u_c_m
+
    tox -r --notest -efull
    # NOTE(mtreinish) Respect constraints in the tempest full venv, things that
    # are using a tox job other than full will not be respecting constraints but
    # running pip install -U on tempest requirements
-    $TEMPEST_DIR/.tox/tempest/bin/pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -r requirements.txt
+    $TEMPEST_DIR/.tox/tempest/bin/pip install -c $tmp_u_c_m -r requirements.txt
    PROJECT_VENV["tempest"]=${TEMPEST_DIR}/.tox/tempest
+    rm -f $tmp_u_c_m
    popd
 }

@@ -671,9 +745,11 @@ function install_tempest {
 function install_tempest_plugins {
    pushd $TEMPEST_DIR
    if [[ $TEMPEST_PLUGINS != 0 ]] ; then
-        # The requirements might be on a different branch, while tempest & tempest plugins needs master requirements.
-        (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > u-c-m.txt
-        tox -evenv-tempest -- pip install -c u-c-m.txt $TEMPEST_PLUGINS
+        local tmp_u_c_m
+        tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
+        set_tempest_venv_constraints $tmp_u_c_m
+        tox -evenv-tempest -- pip install -c $tmp_u_c_m $TEMPEST_PLUGINS
+        rm -f $tmp_u_c_m
        echo "Checking installed Tempest plugins:"
        tox -evenv-tempest -- tempest list-plugins
    fi
@@ -19,6 +19,7 @@
      set_fact:
        external_bridge_mtu: "{{ local_mtu | int - 50 }}"
  roles:
+    - ensure-virtualenv
    - test-matrix
    - configure-swap
    - setup-stack-user
@@ -15,7 +15,7 @@
      name=""
      for u in `systemctl list-unit-files | grep devstack | awk '{print $1}'`; do
        name=$(echo $u | sed 's/devstack@/screen-/' | sed 's/\.service//')
-        journalctl -o short-precise --unit $u | gzip - > {{ stage_dir }}/logs/$name.txt.gz
+        journalctl -o short-precise --unit $u  > {{ stage_dir }}/logs/$name.txt
      done

      # Export the journal in export format to make it downloadable
@@ -34,4 +34,4 @@
          -t sudo \
          --no-pager \
          --since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \
-        | gzip - > {{ stage_dir }}/logs/syslog.txt.gz
+         > {{ stage_dir }}/logs/syslog.txt
@@ -0,0 +1,22 @@
+Generate stackviz report.
+
+Generate stackviz report using subunit and dstat data, using
+the stackviz archive embedded in test images.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :default: /opt/stack
+
+   The devstack base directory.
+
+.. zuul:rolevar:: stage_dir
+   :default: "{{ ansible_user_dir }}"
+
+   The stage directory where the input data can be found and
+   the output will be produced.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ devstack_base_dir }}/tempest
+
+   Directory to work in. It has to be a fully qualified path.
@@ -0,0 +1,3 @@
+devstack_base_dir: /opt/stack
+stage_dir: "{{ ansible_user_dir }}"
+zuul_work_dir: "{{ devstack_base_dir }}/tempest"
@@ -0,0 +1,73 @@
+- name: Process Stackviz
+  block:
+
+  - name: Devstack checks if stackviz archive exists
+    stat:
+      path: "/opt/cache/files/stackviz-latest.tar.gz"
+    register: stackviz_archive
+
+  - debug:
+      msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz"
+    when: not stackviz_archive.stat.exists
+
+  - name: Check if subunit data exists
+    stat:
+      path: "{{ zuul_work_dir }}/testrepository.subunit"
+    register: subunit_input
+
+  - debug:
+      msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit"
+    when: not subunit_input.stat.exists
+
+  - name: Install stackviz
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+    block:
+      - include_role:
+          name: ensure-pip
+
+      - pip:
+          name: "file://{{ stackviz_archive.stat.path }}"
+          virtualenv: /tmp/stackviz
+          virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+          extra_args: -U
+
+  - name: Deploy stackviz static html+js
+    command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+
+  - name: Check if dstat data exists
+    stat:
+      path: "{{ devstack_base_dir }}/logs/dstat-csv.log"
+    register: dstat_input
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+
+  - name: Run stackviz with dstat
+    shell: |
+      cat {{ subunit_input.stat.path }} | \
+        /tmp/stackviz/bin/stackviz-export \
+          --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \
+          --env --stdin \
+          {{ stage_dir }}/stackviz/data
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+      - dstat_input.stat.exists
+
+  - name: Run stackviz without dstat
+    shell: |
+      cat {{ subunit_input.stat.path }} | \
+        /tmp/stackviz/bin/stackviz-export \
+          --env --stdin \
+          {{ stage_dir }}/stackviz/data
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+      - not dstat_input.stat.exists
+
+  ignore_errors: yes
@@ -1,9 +1,13 @@
 - name: Find all OpenStack source repos used by this job
  find:
    paths:
-      - src/git.openstack.org/openstack
-      - src/git.openstack.org/openstack-dev
-      - src/git.openstack.org/openstack-infra
+      - src/opendev.org/opendev
+      - src/opendev.org/openstack
+      - src/opendev.org/openstack-dev
+      - src/opendev.org/openstack-infra
+      - src/opendev.org/starlingx
+      - src/opendev.org/x
+      - src/opendev.org/zuul
    file_type: directory
  register: found_repos

@@ -10,3 +10,10 @@ subnodes.
   :default: /opt/stack

   The devstack base directory.
+
+.. zuul:rolevar:: devstack_data_base_dir
+   :default: {{ devstack_base_dir }}
+
+   The devstack base directory for data/.
+   Useful for example when multiple executions of devstack (i.e. grenade)
+   share the same data directory.
@@ -1 +1,2 @@
 devstack_base_dir: /opt/stack
+devstack_data_base_dir: "{{ devstack_base_dir }}"
@@ -1,7 +1,7 @@
 - name: Ensure the data folder exists
  become: true
  file:
-    path: "{{ devstack_base_dir }}/data"
+    path: "{{ devstack_data_base_dir }}/data"
    state: directory
    owner: stack
    group: stack
@@ -11,7 +11,7 @@
 - name: Ensure the CA folder exists
  become: true
  file:
-    path: "{{ devstack_base_dir }}/data/CA"
+    path: "{{ devstack_data_base_dir }}/data/CA"
    state: directory
    owner: stack
    group: stack
@@ -25,8 +25,8 @@
    dest: "{{ zuul.executor.work_root }}/{{ item | basename }}"
    mode: pull
  with_items:
-    - "{{ devstack_base_dir }}/data/ca-bundle.pem"
-    - "{{ devstack_base_dir }}/data/CA"
+    - "{{ devstack_data_base_dir }}/data/ca-bundle.pem"
+    - "{{ devstack_data_base_dir }}/data/CA"
  when: inventory_hostname == 'controller'

 - name: Push the CA certificate
@@ -34,7 +34,7 @@
  become_user: stack
  synchronize:
    src: "{{ zuul.executor.work_root }}/ca-bundle.pem"
-    dest: "{{ devstack_base_dir }}/data/ca-bundle.pem"
+    dest: "{{ devstack_data_base_dir }}/data/ca-bundle.pem"
    mode: push
  when: 'inventory_hostname in groups["subnode"]|default([])'

@@ -43,6 +43,6 @@
  become_user: stack
  synchronize:
    src: "{{ zuul.executor.work_root }}/CA/"
-    dest: "{{ devstack_base_dir }}/data/"
+    dest: "{{ devstack_data_base_dir }}/data/"
    mode: push
  when: 'inventory_hostname in groups["subnode"]|default([])'
@@ -252,7 +252,11 @@ class LocalConf(object):
        if localrc:
            vg = VarGraph(localrc)
            for k, v in vg.getVars():
-                self.localrc.append('{}={}'.format(k, v))
+                # Avoid double quoting
+                if len(v) and v[0]=='"':
+                    self.localrc.append('{}={}'.format(k, v))
+                else:
+                    self.localrc.append('{}="{}"'.format(k, v))
                if k == 'LIBS_FROM_GIT':
                    lfg = True
                elif k == 'TEMPEST_PLUGINS':
@@ -185,7 +185,25 @@ class TestDevstackLocalConf(unittest.TestCase):
            for line in f:
                if line.startswith('LIBS_FROM_GIT'):
                    lfg = line.strip().split('=')[1]
-        self.assertEqual('oslo.db', lfg)
+        self.assertEqual('"oslo.db"', lfg)
+
+    def test_avoid_double_quote(self):
+        "Test that there a no duplicated quotes"
+        localrc = {'TESTVAR': '"quoted value"'}
+        p = dict(localrc=localrc,
+                 base_services=[],
+                 base_dir='./test',
+                 path=os.path.join(self.tmpdir, 'test.local.conf'),
+                 projects={})
+        lc = self._init_localconf(p)
+        lc.write(p['path'])
+
+        testvar = None
+        with open(p['path']) as f:
+            for line in f:
+                if line.startswith('TESTVAR'):
+                    testvar = line.strip().split('=')[1]
+        self.assertEqual('"quoted value"', testvar)

    def test_plugin_circular_deps(self):
        "Test that plugins with circular dependencies fail"
@@ -265,7 +283,7 @@ class TestDevstackLocalConf(unittest.TestCase):
        lc.write(p['path'])

        tp = self._find_tempest_plugins_value(p['path'])
-        self.assertEqual('someplugin', tp)
+        self.assertEqual('"someplugin"', tp)
        self.assertEqual(len(lc.warnings), 1)


@@ -221,7 +221,7 @@ write_devstack_version

 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``FORCE=yes ./stack``
-if [[ ! ${DISTRO} =~ (xenial|artful|bionic|stretch|jessie|f28|f29|opensuse-42.3|opensuse-15.0|opensuse-tumbleweed|rhel7) ]]; then
+if [[ ! ${DISTRO} =~ (xenial|artful|bionic|stretch|jessie|f28|f29|opensuse-15.2|opensuse-tumbleweed|rhel7) ]]; then
    echo "WARNING: this script has not been tested on $DISTRO"
    if [[ "$FORCE" != "yes" ]]; then
        die $LINENO "If you wish to run this script anyway run with FORCE=yes"
@@ -328,7 +328,13 @@ function _install_rdo {
    # Per the point above, it's a bunch of repos so starts getting a
    # little messy...
    if ! is_package_installed rdo-release ; then
-        yum_install https://rdoproject.org/repos/rdo-release.rpm
+        if [[ "$TARGET_BRANCH" == "master" ]]; then
+            yum_install https://rdoproject.org/repos/rdo-release.rpm
+        else
+            # Get latest rdo-release-$rdo_release RPM package version
+            rdo_release=$(echo $TARGET_BRANCH | sed "s|stable/||g")
+            yum_install https://rdoproject.org/repos/openstack-$rdo_release/rdo-release-$rdo_release.rpm
+        fi
    fi

    # Also enable optional for RHEL7 proper.  Note this is a silent
@@ -767,9 +773,6 @@ if [[ "$OFFLINE" != "True" ]]; then
    PYPI_ALTERNATIVE_URL=${PYPI_ALTERNATIVE_URL:-""} $TOP_DIR/tools/install_pip.sh
 fi

-# Install subunit for the subunit output stream
-pip_install -U os-testr
-
 TRACK_DEPENDS=${TRACK_DEPENDS:-False}

 # Install Python packages into a virtualenv so that we can track them
@@ -787,6 +790,9 @@ fi
 source $TOP_DIR/tools/fixup_stuff.sh
 fixup_all

+# Install subunit for the subunit output stream
+pip_install -U os-testr
+
 if [[ "$USE_SYSTEMD" == "True" ]]; then
    pip_install_gr systemd-python
    # the default rate limit of 1000 messages / 30 seconds is not
@@ -15,7 +15,7 @@ source $RC_DIR/functions

 # Set the target branch. This is used so that stable branching
 # does not need to update each repo below.
-TARGET_BRANCH=master
+TARGET_BRANCH=stable/stein

 # Cycle trailing projects need to branch later than the others.
 TRAILING_TARGET_BRANCH=master
@@ -314,7 +314,11 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH}

 # Tempest test suite
 TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git}
-TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH}
+# Use Tempest tag 24.0.0 which is Stein supported version
+# Tempest 20.0.0 to 26.0.0 are supported version for stable/stein
+# https://docs.openstack.org/releasenotes/tempest/v24.0.0.html
+TEMPEST_BRANCH=${TEMPEST_BRANCH:-24.0.0}
+TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt}


 ##############
@@ -774,6 +778,10 @@ ETCD_DOWNLOAD_LOCATION=$ETCD_DOWNLOAD_URL/$ETCD_VERSION/$ETCD_DOWNLOAD_FILE
 # etcd is always required, so place it into list of pre-cached downloads
 EXTRA_CACHE_URLS+=",$ETCD_DOWNLOAD_LOCATION"

+# Cache settings
+CACHE_BACKEND=${CACHE_BACKEND:-"dogpile.cache.memcached"}
+MEMCACHE_SERVERS=${MEMCACHE_SERVERS:-"localhost:11211"}
+
 # Detect duplicate values in IMAGE_URLS
 for image_url in ${IMAGE_URLS//,/ }; do
    if [ $(echo "$IMAGE_URLS" | grep -o -F "$image_url" | wc -l) -gt 1 ]; then
@@ -95,19 +95,7 @@ function test_libs_exist {
    echo "test_libs_exist PASSED"
 }

-function test_branch_master {
-    for lib in $ALL_LIBS; do
-        if [[ ${GITBRANCH[$lib]} != "master" ]]; then
-            echo "GITBRANCH for $lib not master (${GITBRANCH[$lib]})"
-            exit 1
-        fi
-    done
-
-    echo "test_branch_master PASSED"
-}
-
 set -o errexit

 test_libs_exist
-test_branch_master
 test_all_libs_upto_date
@@ -15,7 +15,7 @@

 echo "Ensuring we don't have crazy refs"

-REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'`
+REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'`
 rc=$?
 if [[ $rc -eq 0 ]]; then
    echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:"
@@ -216,6 +216,20 @@ function fixup_suse {
        sudo systemctl disable apparmor
        sudo /usr/sbin/aa-teardown
    fi
+
+    # Ensure trusted CA certificates are up to date
+    # See https://bugzilla.suse.com/show_bug.cgi?id=1154871
+    # May be removed once a new opensuse-15 image is available in nodepool
+    sudo zypper up -y p11-kit ca-certificates-mozilla
+
+    # Since pip10, pip will refuse to uninstall files from packages
+    # that were created with distutils (rather than more modern
+    # setuptools).  This is because it technically doesn't have a
+    # manifest of what to remove.  However, in most cases, simply
+    # overwriting works.  So this hacks around those packages that
+    # have been dragged in by some other system dependency
+    sudo rm -rf /usr/lib/python3.6/site-packages/ply-*.egg-info
+    sudo rm -rf /usr/lib/python3.6/site-packages/six-*.egg-info
 }

 # The version of pip(1.5.4) supported by python-virtualenv(1.11.4) has
@@ -38,7 +38,9 @@ FILES=$TOP_DIR/files
 # [1] http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/elements/cache-devstack/source-repository-pip

 PIP_GET_PIP_URL=${PIP_GET_PIP_URL:-"https://bootstrap.pypa.io/get-pip.py"}
+PIP_GET_PIP_PY27_URL=${PIP_GET_PIP_PY27_URL:-"https://bootstrap.pypa.io/pip/2.7/get-pip.py"}
 LOCAL_PIP="$FILES/$(basename $PIP_GET_PIP_URL)"
+LOCAL_PIP_PY27="${LOCAL_PIP}-py27"

 GetDistro
 echo "Distro: $DISTRO"
@@ -56,13 +58,17 @@ function get_versions {
 }


-function install_get_pip {
+function _install_get_pip {
+    local interpreter=$1
+    local pip_get_pip_url=$2
+    local local_pip=$3
+
    # If get-pip.py isn't python, delete it. This was probably an
    # outage on the server.
-    if [[ -r $LOCAL_PIP ]]; then
-        if ! head -1 $LOCAL_PIP | grep -q '#!/usr/bin/env python'; then
-            echo "WARNING: Corrupt $LOCAL_PIP found removing"
-            rm $LOCAL_PIP
+    if [[ -r $local_pip ]]; then
+        if ! head -1 $local_pip | grep -q '#!/usr/bin/env python'; then
+            echo "WARNING: Corrupt $local_pip found removing"
+            rm $local_pip
        fi
    fi

@@ -76,22 +82,34 @@ function install_get_pip {
    # Thus we use curl's "-z" feature to always check the modified
    # since and only download if a new version is out -- but only if
    # it seems we downloaded the file originally.
-    if [[ ! -r $LOCAL_PIP || -r $LOCAL_PIP.downloaded ]]; then
+    if [[ ! -r $local_pip || -r $local_pip.downloaded ]]; then
        # only test freshness if LOCAL_PIP is actually there,
        # otherwise we generate a scary warning.
        local timecond=""
-        if [[ -r $LOCAL_PIP ]]; then
-            timecond="-z $LOCAL_PIP"
+        if [[ -r $local_pip ]]; then
+            timecond="-z $local_pip"
        fi

        curl -f --retry 6 --retry-delay 5 \
-            $timecond -o $LOCAL_PIP $PIP_GET_PIP_URL || \
+            $timecond -o $local_pip $pip_get_pip_url || \
            die $LINENO "Download of get-pip.py failed"
-        touch $LOCAL_PIP.downloaded
+        touch $local_pip.downloaded
    fi
-    sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
+    sudo -H -E $interpreter $local_pip -c $TOOLS_DIR/cap-pip.txt
+}
+
+
+function install_get_pip {
+    _install_get_pip python $PIP_GET_PIP_PY27_URL $LOCAL_PIP_PY27
    if python3_enabled; then
-        sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
+        if [[ "$PYTHON3_VERSION" == "3.5" ]]; then
+            PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/pip/3.5/$(basename $PIP_GET_PIP_URL)
+            LOCAL_PIP=${LOCAL_PIP}-py35
+        elif [[ "$PYTHON3_VERSION" == "3.6" ]]; then
+            PIP_GET_PIP_URL=$(dirname $PIP_GET_PIP_URL)/pip/3.6/$(basename $PIP_GET_PIP_URL)
+            LOCAL_PIP=${LOCAL_PIP}-py36
+        fi
+        _install_get_pip python${PYTHON3_VERSION} $PIP_GET_PIP_URL $LOCAL_PIP
    fi
 }

@@ -181,3 +181,9 @@ if is_service_enabled cinder && is_package_installed lvm2; then
    clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME || /bin/true
    clean_lvm_filter
 fi
+
+
+# Clean any safe.directory items we wrote into the global
+# gitconfig. We can identify the relevant ones by checking that they
+# point to somewhere in our $DEST directory.
+sudo sed -i "/directory=${DEST}/ d" /etc/gitconfig