Try to reduce mysql memory usage

These are a few tweaks I applied to my own memory-constrained cloud
instances that seemed to help. I have lower performance requirements
so this may make things worse and not better, but it's worth seeing
what the impact is. I'll admit to not knowing the full impact of these
as they're mostly collected from various tutorials on lowering memory
usage.

Enable this for now on devstack-multinode

Change-Id: I7b223391d3de01e3e81b02076debd01d9d2f097c
(cherry picked from commit 7567359755)
(cherry picked from commit b12cc4181c)
(cherry picked from commit c4a1f86548)
(cherry picked from commit 275a84f929)

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/devstack.git
+defaultbranch=unmaintained/wallaby

.zuul.yaml

@@ -39,20 +39,20 @@
           - controller
 
 - nodeset:
-    name: devstack-single-node-centos-7
+    name: devstack-single-node-centos-8-stream
     nodes:
       - name: controller
-        label: centos-7
+        label: centos-8-stream
     groups:
       - name: tempest
         nodes:
           - controller
 
 - nodeset:
-    name: devstack-single-node-centos-8
+    name: devstack-single-node-centos-9-stream
     nodes:
       - name: controller
-        label: centos-8
+        label: centos-9-stream
     groups:
       - name: tempest
         nodes:
@@ -68,16 +68,6 @@
         nodes:
           - controller
 
-- nodeset:
-    name: devstack-single-node-fedora-latest
-    nodes:
-      - name: controller
-        label: fedora-32
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
 - nodeset:
     name: openstack-two-node
     nodes:
@@ -285,7 +275,6 @@
     required-projects:
       - opendev.org/openstack/devstack
     roles:
-      - zuul: opendev.org/openstack/devstack-gate
       - zuul: opendev.org/openstack/openstack-zuul-jobs
     vars:
       devstack_localrc:
@@ -403,7 +392,7 @@
         PUBLIC_BRIDGE_MTU: '{{ external_bridge_mtu }}'
       devstack_services:
         # Shared services
-        dstat: true
+        dstat: false
         etcd3: true
         memory_tracker: true
         mysql: true
@@ -412,7 +401,7 @@
       subnode:
         devstack_services:
           # Shared services
-          dstat: true
+          dstat: false
           memory_tracker: true
         devstack_localrc:
           # Multinode specific settings
@@ -477,7 +466,7 @@
         # Core services enabled for this branch.
         # This list replaces the test-matrix.
         # Shared services
-        dstat: true
+        dstat: false
         etcd3: true
         memory_tracker: true
         mysql: true
@@ -527,7 +516,7 @@
           # Core services enabled for this branch.
           # This list replaces the test-matrix.
           # Shared services
-          dstat: true
+          dstat: false
           memory_tracker: true
           tls-proxy: true
           # Nova services
@@ -571,25 +560,9 @@
     description: |
       Simple multinode test to verify multinode functionality on devstack side.
       This is not meant to be used as a parent job.
-
-# NOTE(ianw) Platform tests have traditionally been non-voting because
-# we often have to rush things through devstack to stabilise the gate,
-# and these platforms don't have the round-the-clock support to avoid
-# becoming blockers in that situation.
-- job:
-    name: devstack-platform-centos-8
-    parent: tempest-full-py3
-    description: Centos 8 platform test
-    nodeset: devstack-single-node-centos-8
-    voting: false
-    timeout: 9000
-
-- job:
-    name: devstack-platform-bionic
-    parent: tempest-full-py3
-    description: Ubuntu Bionic platform test
-    nodeset: openstack-single-node-bionic
-    voting: false
+    vars:
+      devstack_localrc:
+        MYSQL_REDUCE_MEMORY: true
 
 - job:
     name: devstack-async
@@ -602,23 +575,6 @@
       zuul_copy_output:
         /opt/stack/async: logs
 
-- job:
-    name: devstack-platform-fedora-latest
-    parent: tempest-full-py3
-    description: Fedora latest platform test
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-
-- job:
-    name: devstack-platform-fedora-latest-virt-preview
-    parent: tempest-full-py3
-    description: Fedora latest platform test using the virt-preview repo.
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-    vars:
-      devstack_localrc:
-        ENABLE_FEDORA_VIRT_PREVIEW_REPO: true
-
 - job:
     name: devstack-tox-base
     parent: devstack
@@ -675,6 +631,7 @@
 
 - job:
     name: devstack-unit-tests
+    nodeset: ubuntu-focal
     description: |
       Runs unit tests on devstack project.
 
@@ -690,9 +647,6 @@
       jobs:
         - devstack
         - devstack-ipv6
-        - devstack-platform-fedora-latest
-        - devstack-platform-centos-8
-        - devstack-platform-bionic
         - devstack-async
         - devstack-multinode
         - devstack-unit-tests
@@ -704,10 +658,8 @@
             irrelevant-files: &dsvm-irrelevant-files
               - ^.*\.rst$
               - ^doc/.*$
-        - swift-dsvm-functional-py3:
-            voting: false
-            irrelevant-files: *dsvm-irrelevant-files
         - grenade:
+            voting: false
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
@@ -729,7 +681,10 @@
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
+        # TODO(gmann): make it voting once it is fixed, currently it is failing
+        # for <=stable/xena https://zuul.openstack.org/builds?job_name=openstacksdk-functional-devstack&branch=stable%2Fxena&branch=stable%2Fwallaby&skip=0
         - openstacksdk-functional-devstack:
+            voting: false
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
@@ -738,6 +693,7 @@
               - ^.*\.rst$
               - ^doc/.*$
         - nova-ceph-multistore:
+            voting: false
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
@@ -757,21 +713,26 @@
               - ^.*\.rst$
               - ^doc/.*$
         - grenade:
+            voting: false
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
-        - openstacksdk-functional-devstack:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        # - openstacksdk-functional-devstack:
+        #    voting: false
+        #    irrelevant-files:
+        #      - ^.*\.rst$
+        #      - ^doc/.*$
         - tempest-ipv6-only:
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
-        - nova-ceph-multistore:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
+        # TODO(elod.illes): nova-ceph-multistore is broken in this branch,
+        # so it is set as non-voting in check queue and commented out here
+        # in gate queue until the job gets fixed.
+        # - nova-ceph-multistore:
+        #     irrelevant-files:
+        #       - ^.*\.rst$
+        #       - ^doc/.*$
     # Please add a note on each job and conditions for the job not
     # being experimental any more, so we can keep this list somewhat
     # pruned.
@@ -821,4 +782,3 @@
             irrelevant-files:
               - ^.*\.rst$
               - ^doc/.*$
-        - devstack-platform-fedora-latest-virt-preview

files/rpms/ceph

@@ -1,3 +1,3 @@
 ceph    # NOPRIME
-redhat-lsb-core
+redhat-lsb-core # not:rhel9
 xfsprogs

files/rpms/n-cpu

@@ -1,9 +1,10 @@
 cryptsetup
 dosfstools
-genisoimage
+genisoimage # not:rhel9
 iscsi-initiator-utils
 libosinfo
 lvm2
 sg3_utils
 # Stuff for diablo volumes
 sysfsutils
+xorriso # not:rhel8

files/rpms/nova

@@ -4,7 +4,7 @@ dnsmasq # for q-dhcp
 dnsmasq-utils # for dhcp_release
 ebtables
 gawk
-genisoimage # required for config_drive
+genisoimage # not:rhel9 required for config_drive
 iptables
 iputils
 kernel-modules
@@ -14,3 +14,4 @@ polkit
 rabbitmq-server # NOPRIME
 sqlite
 sudo
+xorriso # not:rhel8

files/rpms/swift

@@ -4,4 +4,4 @@ memcached
 rsync-daemon
 sqlite
 xfsprogs
-xinetd
+xinetd # not:f34,rhel9

functions-common

@@ -346,12 +346,19 @@ function _ensure_lsb_release {
 #  - os_VENDOR
 #  - os_PACKAGE
 function GetOSVersion {
-    # We only support distros that provide a sane lsb_release
-    _ensure_lsb_release
+    # CentOS Stream 9 does not provide lsb_release
+    source /etc/os-release
+    if [[ "${ID}${VERSION}" == "centos9" ]]; then
+        os_RELEASE=${VERSION_ID}
+        os_CODENAME="n/a"
+        os_VENDOR=$(echo $NAME | tr -d '[:space:]')
+    else
+        _ensure_lsb_release
 
-    os_RELEASE=$(lsb_release -r -s)
-    os_CODENAME=$(lsb_release -c -s)
-    os_VENDOR=$(lsb_release -i -s)
+        os_RELEASE=$(lsb_release -r -s)
+        os_CODENAME=$(lsb_release -c -s)
+        os_VENDOR=$(lsb_release -i -s)
+    fi
 
     if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
         os_PACKAGE="deb"
@@ -588,6 +595,18 @@ function git_clone {
         fi
     fi
 
+    # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
+    # about how we clone and work with repos.  Mark them safe globally
+    # as a work-around.
+    #
+    # NOTE(danms): On bionic (and likely others) git-config may write
+    # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+    # writes these changes to /etc/gitconfig which is more
+    # discoverable anyway.
+    #
+    # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
+    sudo git config --system --add safe.directory ${git_dest}
+
     # print out the results so we know what change was used in the logs
     cd $git_dest
     git show --oneline | head -1
@@ -1442,6 +1461,7 @@ function write_user_unit_file {
     local command="$2"
     local group=$3
     local user=$4
+    local env_vars="$5"
     local extra=""
     if [[ -n "$group" ]]; then
         extra="Group=$group"
@@ -1455,6 +1475,9 @@ function write_user_unit_file {
     iniset -sudo $unitfile "Service" "KillMode" "process"
     iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
     iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
     if [[ -n "$group" ]]; then
         iniset -sudo $unitfile "Service" "Group" "$group"
     fi
@@ -1469,6 +1492,7 @@ function write_uwsgi_user_unit_file {
     local command="$2"
     local group=$3
     local user=$4
+    local env_vars="$5"
     local unitfile="$SYSTEMD_DIR/$service"
     mkdir -p $SYSTEMD_DIR
 
@@ -1483,6 +1507,9 @@ function write_uwsgi_user_unit_file {
     iniset -sudo $unitfile "Service" "NotifyAccess" "all"
     iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"
 
+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
     if [[ -n "$group" ]]; then
         iniset -sudo $unitfile "Service" "Group" "$group"
     fi
@@ -1530,10 +1557,14 @@ function _run_under_systemd {
     local systemd_service="devstack@$service.service"
     local group=$3
     local user=${4:-$STACK_USER}
+    if [[ -z "$user" ]]; then
+        user=$STACK_USER
+    fi
+    local env_vars="$5"
     if [[ "$command" =~ "uwsgi" ]] ; then
-        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
     else
-        write_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
     fi
 
     $SYSTEMCTL enable $systemd_service
@@ -1554,18 +1585,20 @@ function is_running {
 # If the command includes shell metachatacters (;<>*) it must be run using a shell
 # If an optional group is provided sg will be used to run the
 # command as that group.
-# run_process service "command-line" [group] [user]
+# run_process service "command-line" [group] [user] [env_vars]
+# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2"
 function run_process {
     local service=$1
     local command="$2"
     local group=$3
     local user=$4
+    local env_vars="$5"
 
     local name=$service
 
     time_start "run_process"
     if is_service_enabled $service; then
-        _run_under_systemd "$name" "$command" "$group" "$user"
+        _run_under_systemd "$name" "$command" "$group" "$user" "$env_vars"
     fi
     time_stop "run_process"
 }

inc/python

@@ -378,12 +378,13 @@ function _setup_package_with_constraints_edit {
     project_dir=$(cd $project_dir && pwd)
 
     if [ -n "$REQUIREMENTS_DIR" ]; then
-        # Constrain this package to this project directory from here on out.
+        # Remove this package from constraints before we install it.
+        # That way, later installs won't "downgrade" the install from
+        # source we are about to do.
         local name
         name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
         $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
-            $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
-            "$flags file://$project_dir#egg=$name"
+            $REQUIREMENTS_DIR/upper-constraints.txt -- $name
     fi
 
     setup_package $bindep $project_dir "$flags" $extras

lib/apache

@@ -306,7 +306,7 @@ function write_uwsgi_config {
         apache_conf=$(apache_site_config_for $name)
         iniset "$file" uwsgi socket "$socket"
         iniset "$file" uwsgi chmod-socket 666
-        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee -a $apache_conf
+        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 " | sudo tee -a $apache_conf
         enable_apache_site $name
         restart_apache_server
     fi

lib/cinder

@@ -31,6 +31,7 @@ set +o xtrace
 CINDER_DRIVER=${CINDER_DRIVER:-default}
 CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins
 CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends
+CINDER_BACKUPS=$TOP_DIR/lib/cinder_backups
 
 # grab plugin config if specified via cinder_driver
 if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
@@ -98,6 +99,16 @@ else
     CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-tgtadm}
 fi
 
+# For backward compatibility
+# Before CINDER_BACKUP_DRIVER was introduced, ceph backup driver was configured
+# along with ceph backend driver.
+if [[ -z "${CINDER_BACKUP_DRIVER}" && "$CINDER_ENABLED_BACKENDS" =~ "ceph" ]]; then
+    CINDER_BACKUP_DRIVER=ceph
+fi
+
+# Supported backup drivers are in lib/cinder_backups
+CINDER_BACKUP_DRIVER=${CINDER_BACKUP_DRIVER:-swift}
+
 # Toggle for deploying Cinder under a wsgi server. Legacy mod_wsgi
 # reference should be cleaned up to more accurately refer to uwsgi.
 CINDER_USE_MOD_WSGI=${CINDER_USE_MOD_WSGI:-True}
@@ -113,6 +124,15 @@ if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
     done
 fi
 
+# Source the backup driver
+if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+    if [[ -r $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER ]]; then
+        source $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER
+    else
+        die "cinder backup driver $CINDER_BACKUP_DRIVER is not supported"
+    fi
+fi
+
 # Environment variables to configure the image-volume cache
 CINDER_IMG_CACHE_ENABLED=${CINDER_IMG_CACHE_ENABLED:-True}
 
@@ -189,6 +209,12 @@ function cleanup_cinder {
         done
     fi
 
+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type cleanup_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            cleanup_cinder_backup_$CINDER_BACKUP_DRIVER
+        fi
+    fi
+
     stop_process "c-api"
     remove_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI"
 }
@@ -266,13 +292,12 @@ function configure_cinder {
         configure_cinder_image_volume_cache
     fi
 
-    if is_service_enabled c-bak; then
-        # NOTE(mriedem): The default backup driver uses swift and if we're
-        # on a subnode we might not know if swift is enabled, but chances are
-        # good that it is on the controller so configure the backup service
-        # to use it. If we want to configure the backup service to use
-        # a non-swift driver, we'll likely need environment variables.
-        iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type configure_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            configure_cinder_backup_$CINDER_BACKUP_DRIVER
+        else
+            die "configure_cinder_backup_$CINDER_BACKUP_DRIVER doesn't exist in $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER"
+        fi
     fi
 
     if is_service_enabled ceilometer; then
@@ -410,6 +435,12 @@ function init_cinder {
         done
     fi
 
+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type init_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            init_cinder_backup_$CINDER_BACKUP_DRIVER
+        fi
+    fi
+
     mkdir -p $CINDER_STATE_PATH/volumes
 }
 
@@ -504,8 +535,13 @@ function start_cinder {
     fi
 
     run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
-    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
-    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
+    # Tune glibc for Python Services using single malloc arena for all threads
+    # and disabling dynamic thresholds to reduce memory usage when using native
+    # threads directly or via eventlet.tpool
+    # https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html
+    malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144"
+    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning"
+    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning"
 
     # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
     # by the scheduler start the cinder-volume service last (or restart it) after the scheduler

lib/cinder_backends/ceph

@@ -6,12 +6,6 @@
 # Enable with:
 #
 #   CINDER_ENABLED_BACKENDS+=,ceph:ceph
-#
-# Optional parameters:
-#   CINDER_BAK_CEPH_POOL=<pool-name>
-#   CINDER_BAK_CEPH_USER=<user>
-#   CINDER_BAK_CEPH_POOL_PG=<pg-num>
-#   CINDER_BAK_CEPH_POOL_PGP=<pgp-num>
 
 # Dependencies:
 #
@@ -29,11 +23,6 @@ set +o xtrace
 # Defaults
 # --------
 
-CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups}
-CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8}
-CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8}
-CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}
-
 
 # Entry Points
 # ------------
@@ -52,27 +41,6 @@ function configure_cinder_backend_ceph {
     iniset $CINDER_CONF $be_name rbd_flatten_volume_from_snapshot False
     iniset $CINDER_CONF $be_name rbd_max_clone_depth 5
     iniset $CINDER_CONF DEFAULT glance_api_version 2
-
-    if is_service_enabled c-bak; then
-        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
-        if [ "$REMOTE_CEPH" = "False" ]; then
-            # Configure Cinder backup service options, ceph pool, ceph user and ceph key
-            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} size ${CEPH_REPLICAS}
-            if [[ $CEPH_REPLICAS -ne 1 ]]; then
-                sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
-            fi
-        fi
-        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-
-        iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
-        iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
-        iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL"
-        iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER"
-        iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0
-        iniset $CINDER_CONF DEFAULT backup_ceph_stripe_count 0
-        iniset $CINDER_CONF DEFAULT restore_discard_excess_bytes True
-    fi
 }
 
 # Restore xtrace

lib/cinder_backups/ceph

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# lib/cinder_backups/ceph
+# Configure the ceph backup driver
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=ceph
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_CEPH=$(set +o | grep xtrace)
+set +o xtrace
+
+# Defaults
+# --------
+
+CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups}
+CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8}
+CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8}
+CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}
+
+
+function configure_cinder_backup_ceph {
+    sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
+    if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+    fi
+    sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+    sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
+    iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL"
+    iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER"
+    iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0
+    iniset $CINDER_CONF DEFAULT backup_ceph_stripe_count 0
+    iniset $CINDER_CONF DEFAULT restore_discard_excess_bytes True
+}
+
+# init_cinder_backup_ceph: nothing to do
+# cleanup_cinder_backup_ceph: nothing to do
+
+# Restore xtrace
+$_XTRACE_CINDER_CEPH
+
+# Local variables:
+# mode: shell-script
+# End:

lib/cinder_backups/s3_swift

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# lib/cinder_backups/s3_swift
+# Configure the s3 backup driver with swift s3api
+#
+# TODO: create lib/cinder_backup/s3 for external s3 compatible storage
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=s3_swift
+#   enable_service s3api s-proxy s-object s-container s-account
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_S3_SWIFT=$(set +o | grep xtrace)
+set +o xtrace
+
+function configure_cinder_backup_s3_swift {
+    # This configuration requires swift and s3api. If we're
+    # on a subnode we might not know if they are enabled
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.s3.S3BackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_s3_endpoint_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$S3_SERVICE_PORT"
+}
+
+function init_cinder_backup_s3_swift {
+    openstack ec2 credential create
+    iniset $CINDER_CONF DEFAULT backup_s3_store_access_key "$(openstack ec2 credential list -c Access -f value)"
+    iniset $CINDER_CONF DEFAULT backup_s3_store_secret_key "$(openstack ec2 credential list -c Secret -f value)"
+    if is_service_enabled tls-proxy; then
+        iniset $CINDER_CONF DEFAULT backup_s3_ca_cert_file "$SSL_BUNDLE_FILE"
+    fi
+}
+
+# cleanup_cinder_backup_s3_swift: nothing to do
+
+# Restore xtrace
+$_XTRACE_CINDER_S3_SWIFT
+
+# Local variables:
+# mode: shell-script
+# End:

lib/cinder_backups/swift

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# lib/cinder_backups/swift
+# Configure the swift backup driver
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=swift
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_SWIFT=$(set +o | grep xtrace)
+set +o xtrace
+
+
+function configure_cinder_backup_swift {
+    # NOTE(mriedem): The default backup driver uses swift and if we're
+    # on a subnode we might not know if swift is enabled, but chances are
+    # good that it is on the controller so configure the backup service
+    # to use it.
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.swift.SwiftBackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+}
+
+# init_cinder_backup_swift: nothing to do
+# cleanup_cinder_backup_swift: nothing to do
+
+
+# Restore xtrace
+$_XTRACE_CINDER_SWIFT
+
+# Local variables:
+# mode: shell-script
+# End:

lib/databases/mysql

@@ -143,6 +143,16 @@ function configure_database_mysql {
         iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1
     fi
 
+    if [[ "$MYSQL_REDUCE_MEMORY" == "True" ]]; then
+        iniset -sudo $my_conf mysqld read_buffer_size 64K
+        iniset -sudo $my_conf mysqld innodb_buffer_pool_size 16M
+        iniset -sudo $my_conf mysqld thread_stack 192K
+        iniset -sudo $my_conf mysqld thread_cache_size 8
+        iniset -sudo $my_conf mysqld tmp_table_size 8M
+        iniset -sudo $my_conf mysqld sort_buffer_size 8M
+        iniset -sudo $my_conf mysqld max_allowed_packet 8M
+    fi
+
     restart_service $MYSQL_SERVICE_NAME
 }
 

lib/glance

@@ -85,6 +85,12 @@ GLANCE_TASKS_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_tasks_s
 
 GLANCE_USE_IMPORT_WORKFLOW=$(trueorfalse False GLANCE_USE_IMPORT_WORKFLOW)
 
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the Image API policies to start checking the scope of token. By Default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+GLANCE_ENFORCE_SCOPE=$(trueorfalse False GLANCE_ENFORCE_SCOPE)
+
 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
 GLANCE_API_CONF=$GLANCE_CONF_DIR/glance-api.conf
@@ -373,6 +379,12 @@ function configure_glance {
         iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
         iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
     fi
+
+    if [[ "$GLANCE_ENFORCE_SCOPE" == True ]] ; then
+        iniset $GLANCE_API_CONF oslo_policy enforce_scope true
+        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults true
+        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac true
+    fi
 }
 
 # create_glance_accounts() - Set up common required glance accounts

lib/neutron-legacy

@@ -875,6 +875,9 @@ function _configure_neutron_service {
 
     configure_keystone_authtoken_middleware $NEUTRON_CONF nova nova
 
+    # Configuration for placement client
+    configure_keystone_authtoken_middleware $NEUTRON_CONF placement placement
+
     # Configure plugin
     neutron_plugin_configure_service
 }

lib/nova

@@ -304,6 +304,9 @@ function configure_nova {
             sudo dnf update -y
         fi
 
+        # Ensure each compute host uses a unique iSCSI initiator
+        echo InitiatorName=$(iscsi-iname) | sudo tee /etc/iscsi/initiatorname.iscsi
+
         if [[ ${ISCSID_DEBUG} == "True" ]]; then
             # Install an override that starts iscsid with debugging
             # enabled.
@@ -317,6 +320,14 @@ EOF
             sudo systemctl daemon-reload
         fi
 
+        # set chap algorithms.  The default chap_algorithm is md5 which will
+        # not work under FIPS.
+        # FIXME(alee) For some reason, this breaks openeuler.  Openeuler devs should weigh in
+        # and determine the correct solution for openeuler here
+        if ! is_openeuler; then
+            iniset -sudo /etc/iscsi/iscsid.conf DEFAULT "node.session.auth.chap_algs" "SHA3-256,SHA256"
+        fi
+
         # ensure that iscsid is started, even when disabled by default
         restart_service iscsid
     fi
@@ -481,7 +492,8 @@ function create_nova_conf {
     fi
 
     # nova defaults to genisoimage but only mkisofs is available for 15.0+
-    if is_suse; then
+    # rhel provides mkisofs symlink to genisoimage or xorriso appropiately
+    if is_suse || is_fedora; then
         iniset $NOVA_CONF DEFAULT mkisofs_cmd /usr/bin/mkisofs
     fi
 
@@ -932,6 +944,11 @@ function start_nova_compute {
         iniset $NOVA_CPU_CONF os_vif_ovs ovsdb_connection "tcp:$OVSDB_SERVER_LOCAL_HOST:6640"
     fi
 
+    # Workaround bug #1939108
+    if [[ "$VIRT_DRIVER" == "libvirt" && "$LIBVIRT_TYPE" == "qemu" ]]; then
+        iniset $NOVA_CPU_CONF workarounds libvirt_disable_apic True
+    fi
+
     if [[ "$VIRT_DRIVER" = 'libvirt' ]]; then
         # The group **$LIBVIRT_GROUP** is added to the current user in this script.
         # ``sg`` is used in run_process to execute nova-compute as a member of the

lib/nova_plugins/functions-libvirt

@@ -56,15 +56,23 @@ EOF
 
 # Installs required distro-specific libvirt packages.
 function install_libvirt {
+    # NOTE(yoctozepto): The common consensus [1] is that libvirt-python should
+    # be installed from distro packages. However, various projects might be
+    # trying to ensure it is installed using pip AND use upper-constraints
+    # with that, causing pip to try to upgrade it and to fail.
+    # The following line removes libvirt-python from upper-constraints and
+    # avoids the situation described above. Now only if installed packages
+    # explicitly depend on a newer (or, in general, incompatible) libvirt-python
+    # version, will pip try to reinstall it.
+    # [1] https://review.opendev.org/c/openstack/devstack/+/798514
+    $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
+            $REQUIREMENTS_DIR/upper-constraints.txt -- libvirt-python
 
     if is_ubuntu; then
-        install_package qemu-system libvirt-clients libvirt-daemon-system libvirt-dev
+        install_package qemu-system libvirt-clients libvirt-daemon-system libvirt-dev python3-libvirt
         if is_arch "aarch64"; then
             install_package qemu-efi
         fi
-        # uninstall in case the libvirt version changed
-        pip_uninstall libvirt-python
-        pip_install_gr libvirt-python
         #pip_install_gr <there-si-no-guestfs-in-pypi>
     elif is_fedora || is_suse; then
 
@@ -79,14 +87,11 @@ function install_libvirt {
         # as the base system version is too old.  We should have
         # pre-installed these
         install_package qemu-kvm
+        install_package libvirt libvirt-devel python3-libvirt
 
-        install_package libvirt libvirt-devel
         if is_arch "aarch64"; then
             install_package edk2.git-aarch64
         fi
-
-        pip_uninstall libvirt-python
-        pip_install_gr libvirt-python
     fi
 
     if [[ $DEBUG_LIBVIRT_COREDUMPS == True ]]; then

lib/swift

@@ -335,7 +335,6 @@ function configure_swift {
     local node_number
     local swift_node_config
     local swift_log_dir
-    local user_group
 
     # Make sure to kill all swift processes first
     $SWIFT_BIN_DIR/swift-init --run-dir=${SWIFT_DATA_DIR}/run all stop || true
@@ -353,7 +352,7 @@ function configure_swift {
     # partitions (which make more sense when you have a multi-node
     # setup) we configure it with our version of rsync.
     sed -e "
-        s/%GROUP%/${USER_GROUP}/;
+        s/%GROUP%/$(id -g -n ${STACK_USER})/;
         s/%USER%/${STACK_USER}/;
         s,%SWIFT_DATA_DIR%,$SWIFT_DATA_DIR,;
     " $FILES/swift/rsyncd.conf | sudo tee /etc/rsyncd.conf

lib/tempest

@@ -58,7 +58,7 @@ BUILD_TIMEOUT=${BUILD_TIMEOUT:-196}
 # This must be False on stable branches, as master tempest
 # deps do not match stable branch deps. Set this to True to
 # have tempest installed in DevStack by default.
-INSTALL_TEMPEST=${INSTALL_TEMPEST:-"True"}
+INSTALL_TEMPEST=${INSTALL_TEMPEST:-"False"}
 
 # This variable is passed directly to pip install inside the common tox venv
 # that is created
@@ -115,7 +115,16 @@ function set_tempest_venv_constraints {
     local tmp_c
     tmp_c=$1
     if [[ $TEMPEST_VENV_UPPER_CONSTRAINTS == "master" ]]; then
-        (cd $REQUIREMENTS_DIR && git show origin/master:upper-constraints.txt) > $tmp_c
+        (cd $REQUIREMENTS_DIR &&
+            git show master:upper-constraints.txt 2>/dev/null ||
+            git show origin/master:upper-constraints.txt) > $tmp_c
+        # NOTE(gmann): we need to set the below env var pointing to master
+        # constraints even that is what default in tox.ini. Otherwise it can
+        # create the issue for grenade run where old and new devstack can have
+        # different tempest (old and master) to install. For detail problem,
+        # refer to the https://bugs.launchpad.net/devstack/+bug/2003993
+        export UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/master
+        export TOX_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/master
     else
         echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env."
         cat $TEMPEST_VENV_UPPER_CONSTRAINTS > $tmp_c
@@ -393,7 +402,7 @@ function configure_tempest {
     # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_compute_max_microversion"
     #       for stable branch on each release which should be changed from "latest" to max supported version of that release.
     local tempest_compute_min_microversion=${TEMPEST_COMPUTE_MIN_MICROVERSION:-None}
-    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"latest"}
+    local tempest_compute_max_microversion=${TEMPEST_COMPUTE_MAX_MICROVERSION:-"2.88"}
     # Reset microversions to None where v2.0 is running which does not support microversion.
     # Both "None" means no microversion testing.
     if [[ "$TEMPEST_COMPUTE_TYPE" == "compute_legacy" ]]; then
@@ -488,7 +497,7 @@ function configure_tempest {
     fi
     iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT)
     local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None}
-    local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"}
+    local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"3.64"}
     # Reset microversions to None where v2 is running which does not support microversion.
     # Both "None" means no microversion testing.
     if [[ "$TEMPEST_VOLUME_TYPE" == "volumev2" ]]; then
@@ -549,7 +558,7 @@ function configure_tempest {
     # NOTE- To avoid microversion tests failure on stable branch, we need to change "tempest_placement_max_microversion"
     #       for stable branch on each release which should be changed from "latest" to max supported version of that release.
     local tempest_placement_min_microversion=${TEMPEST_PLACEMENT_MIN_MICROVERSION:-None}
-    local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"latest"}
+    local tempest_placement_max_microversion=${TEMPEST_PLACEMENT_MAX_MICROVERSION:-"1.36"}
     if [ "$tempest_placement_min_microversion" == "None" ]; then
         inicomment $TEMPEST_CONFIG placement min_microversion
     else
@@ -610,6 +619,8 @@ function configure_tempest {
         fi
     done
 
+    iniset $TEMPEST_CONFIG enforce_scope glance "$GLANCE_ENFORCE_SCOPE"
+
     if [ "$VIRT_DRIVER" = "libvirt" ] && [ "$LIBVIRT_TYPE" = "lxc" ]; then
         # libvirt-lxc does not support boot from volume or attaching volumes
         # so basically anything with cinder is out of the question.
@@ -623,13 +634,13 @@ function configure_tempest {
     local tmp_cfg_file
     tmp_cfg_file=$(mktemp)
     cd $TEMPEST_DIR
-    if [[ "$OFFLINE" != "True" ]]; then
-        tox -revenv-tempest --notest
-    fi
 
     local tmp_u_c_m
     tmp_u_c_m=$(mktemp -t tempest_u_c_m.XXXXXXXXXX)
     set_tempest_venv_constraints $tmp_u_c_m
+    if [[ "$OFFLINE" != "True" ]]; then
+        tox -revenv-tempest --notest
+    fi
     tox -evenv-tempest -- pip install -c $tmp_u_c_m -r requirements.txt
     rm -f $tmp_u_c_m
 
@@ -666,7 +677,25 @@ function configure_tempest {
         DISABLE_NETWORK_API_EXTENSIONS+=", l3_agent_scheduler"
     fi
 
-    local network_api_extensions=${NETWORK_API_EXTENSIONS:-"all"}
+    DEFAULT_NET_EXT="address-scope,agent,allowed-address-pairs,auto-allocated-topology"
+    DEFAULT_NET_EXT+=",availability_zone,binding,default-subnetpools,dhcp_agent_scheduler"
+    DEFAULT_NET_EXT+=",dvr,ext-gw-mode,external-net,extra_dhcp_opt,extraroute,flavors"
+    DEFAULT_NET_EXT+=",l3-flavors,l3-ha,l3_agent_scheduler,multi-provider,net-mtu"
+    DEFAULT_NET_EXT+=",network-ip-availability,network_availability_zone,pagination"
+    DEFAULT_NET_EXT+=",port-security,project-id,provider,quotas,quota_details,rbac-policies"
+    DEFAULT_NET_EXT+=",revision-if-match,router,router_availability_zone,security-group,service-type,sorting"
+    DEFAULT_NET_EXT+=",standard-attr-description,standard-attr-revisions,standard-attr-tag,standard-attr-timestamp"
+    DEFAULT_NET_EXT+=",subnet-service-types,subnet_allocation,net-mtu-writable,ip-substring-filtering"
+    DEFAULT_NET_EXT+=",availability_zone_filter,filter-validation,empty-string-filtering,port-mac-address-regenerate"
+    DEFAULT_NET_EXT+=",port-security-groups-filtering,fip-port-details,binding-extended"
+    DEFAULT_NET_EXT+=",subnet_onboard,l3-port-ip-change-not-allowed,agent-resources-synced"
+    DEFAULT_NET_EXT+=",floatingip-pools,rbac-security-groups,subnetpool-prefix-ops,router-admin-state-down-before-update"
+    DEFAULT_NET_EXT+=",rbac-subnetpool,tag-ports-during-bulk-creation,stateful-security-group,address-group,extraroute-atomic"
+    DEFAULT_NET_EXT+=",port-numa-affinity-policy,rbac-address-scope,security-groups-remote-address-group"
+    #New in wallaby
+    DEFAULT_NET_EXT+=",rbac-address-group,port-device-profile"
+
+    local network_api_extensions=${NETWORK_API_EXTENSIONS:-$DEFAULT_NET_EXT}
     if [[ ! -z "$DISABLE_NETWORK_API_EXTENSIONS" ]]; then
         # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
         network_api_extensions=${NETWORK_API_EXTENSIONS:-$(iniget $tmp_cfg_file network-feature-enabled api_extensions | tr -d " ")}
@@ -678,7 +707,10 @@ function configure_tempest {
     fi
     iniset $TEMPEST_CONFIG network-feature-enabled api_extensions $network_api_extensions
     # Swift API Extensions
-    local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-"all"}
+    DEFAULT_SWIFT_OPT="account_quotas,bulk_delete,bulk_upload,container_quotas"
+    DEFAULT_SWIFT_OPT+=",container_sync,crossdomain,formpost,ratelimit,slo"
+    DEFAULT_SWIFT_OPT+=",staticweb,tempauth,tempurl,versioned_writes"
+    local object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$DEFAULT_SWIFT_OPT}
     if [[ ! -z "$DISABLE_OBJECT_STORAGE_API_EXTENSIONS" ]]; then
         # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
         object_storage_api_extensions=${OBJECT_STORAGE_API_EXTENSIONS:-$(iniget $tmp_cfg_file object-storage-feature-enabled discoverable_apis | tr -d " ")}
@@ -687,7 +719,19 @@ function configure_tempest {
     fi
     iniset $TEMPEST_CONFIG object-storage-feature-enabled discoverable_apis $object_storage_api_extensions
     # Cinder API Extensions
-    local volume_api_extensions=${VOLUME_API_EXTENSIONS:-"all"}
+    DEFAULT_VOL_EXT="OS-SCH-HNT,backups,capabilities,cgsnapshots,consistencygroups"
+    DEFAULT_VOL_EXT+=",encryption,os-admin-actions,os-availability-zone"
+    DEFAULT_VOL_EXT+=",os-extended-services,os-extended-snapshot-attributes"
+    DEFAULT_VOL_EXT+=",os-hosts,os-quota-class-sets,os-quota-sets"
+    DEFAULT_VOL_EXT+=",os-services,os-snapshot-actions,os-snapshot-manage"
+    DEFAULT_VOL_EXT+=",os-snapshot-unmanage,os-types-extra-specs,os-types-manage"
+    DEFAULT_VOL_EXT+=",os-used-limits,os-vol-host-attr,os-vol-image-meta"
+    DEFAULT_VOL_EXT+=",os-vol-mig-status-attr,os-vol-tenant-attr,os-volume-actions"
+    DEFAULT_VOL_EXT+=",os-volume-encryption-metadata,os-volume-manage"
+    DEFAULT_VOL_EXT+=",os-volume-transfer,os-volume-type-access"
+    DEFAULT_VOL_EXT+=",os-volume-unmanage,qos-specs,scheduler-stats"
+
+    local volume_api_extensions=${VOLUME_API_EXTENSIONS:-$DEFAULT_VOL_EXT}
     if [[ ! -z "$DISABLE_VOLUME_API_EXTENSIONS" ]]; then
         # Enabled extensions are either the ones explicitly specified or those available on the API endpoint
         volume_api_extensions=${VOLUME_API_EXTENSIONS:-$(iniget $tmp_cfg_file volume-feature-enabled api_extensions | tr -d " ")}
@@ -703,7 +747,12 @@ function configure_tempest {
 # install_tempest() - Collect source and prepare
 function install_tempest {
     git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH
-    pip_install 'tox!=2.8.0'
+    # NOTE(gmann): Pinning tox<4.0.0 for stable/zed and lower. Tox 4.0.0
+    # released after zed was released and has some incompatible changes
+    # and it is ok not to fix the issues caused by tox 4.0.0 in stable
+    # beanches jobs. We can continue testing the stable/zed and lower
+    # branches with tox<4.0.0
+    pip_install 'tox!=2.8.0,<4.0.0'
     pushd $TEMPEST_DIR
     # NOTE(gmann): checkout the TEMPEST_BRANCH in case TEMPEST_BRANCH
     # is tag name not master. git_clone would not checkout tag because
@@ -715,9 +764,6 @@ function install_tempest {
     set_tempest_venv_constraints $tmp_u_c_m
 
     tox -r --notest -efull
-    # TODO: remove the trailing pip constraint when a proper fix
-    # arrives for bug https://bugs.launchpad.net/devstack/+bug/1906322
-    $TEMPEST_DIR/.tox/tempest/bin/pip install -U -r $RC_DIR/tools/cap-pip.txt
     # NOTE(mtreinish) Respect constraints in the tempest full venv, things that
     # are using a tox job other than full will not be respecting constraints but
     # running pip install -U on tempest requirements

lib/tls

@@ -169,7 +169,7 @@ default_md              = default
 
 [ req ]
 default_bits            = 1024
-default_md              = sha1
+default_md              = sha256
 
 prompt                  = no
 distinguished_name      = req_distinguished_name
@@ -261,7 +261,7 @@ function make_cert {
     if [ ! -r "$ca_dir/$cert_name.crt" ]; then
         # Generate a signing request
         $OPENSSL req \
-            -sha1 \
+            -sha256 \
             -newkey rsa \
             -nodes \
             -keyout $ca_dir/private/$cert_name.key \
@@ -301,7 +301,7 @@ function make_int_CA {
     if [ ! -r "$ca_dir/cacert.pem" ]; then
         # Create a signing certificate request
         $OPENSSL req -config $ca_dir/ca.conf \
-            -sha1 \
+            -sha256 \
             -newkey rsa \
             -nodes \
             -keyout $ca_dir/private/cacert.key \

playbooks/tox/pre.yaml

@@ -5,4 +5,10 @@
       bindep_profile: test
       bindep_dir: "{{ zuul_work_dir }}"
     - test-setup
-    - ensure-tox
+    # NOTE(gmann): Pinning tox<4.0.0 for stable/zed and lower. Tox 4.0.0
+    # released after zed was released and has some incompatible changes
+    # and it is ok not to fix the issues caused by tox 4.0.0 in stable
+    # beanches jobs. We can continue testing the stable/zed and lower
+    # branches with tox<4.0.0
+    - role: ensure-tox
+      ensure_tox_version: "<4"

playbooks/tox/run-both.yaml

@@ -6,6 +6,12 @@
       bindep_profile: test
       bindep_dir: "{{ zuul_work_dir }}"
     - test-setup
-    - ensure-tox
+    # NOTE(gmann): Pinning tox<4.0.0 for stable/zed and lower. Tox 4.0.0
+    # released after zed was released and has some incompatible changes
+    # and it is ok not to fix the issues caused by tox 4.0.0 in stable
+    # beanches jobs. We can continue testing the stable/zed and lower
+    # branches with tox<4.0.0
+    - role: ensure-tox
+      ensure_tox_version: "<4"
     - get-devstack-os-environment
     - tox

roles/devstack-ipv6-only-deployments-verification/README.rst

@@ -0,0 +1,16 @@
+Verify the IPv6-only deployments
+
+This role needs to be invoked from a playbook that
+run tests. This role verifies the IPv6 setting on
+devstack side and devstack deploy services on IPv6.
+This role is invoked before tests are run so that
+if any missing IPv6 setting or deployments can fail
+the job early.
+
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :default: /opt/stack
+
+   The devstack base directory.

roles/devstack-ipv6-only-deployments-verification/defaults/main.yaml

@@ -0,0 +1 @@
+devstack_base_dir: /opt/stack

roles/devstack-ipv6-only-deployments-verification/tasks/main.yaml

@@ -0,0 +1,4 @@
+- name: Verify the ipv6-only deployments
+  become: true
+  become_user: stack
+  shell: "{{ devstack_base_dir }}/devstack/tools/verify-ipv6-only-deployments.sh"

roles/export-devstack-journal/tasks/main.yaml

@@ -45,7 +45,7 @@
     cmd: |
       journalctl -o export \
           --since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \
-        | xz --threads=0 - > {{ stage_dir }}/logs/devstack.journal.xz
+        | gzip > {{ stage_dir }}/logs/devstack.journal.gz
 
 - name: Save journal README
   become: true

roles/export-devstack-journal/templates/devstack.journal.README.txt.j2

@@ -7,7 +7,7 @@ devstack run.
 To use it, you will need to convert it so journalctl can read it
 locally.  After downloading the file:
 
- $ /lib/systemd/systemd-journal-remote <(xzcat ./devstack.journal.xz) -o output.journal
+ $ /lib/systemd/systemd-journal-remote <(zcat ./devstack.journal.gz) -o output.journal
 
 Note this binary is not in the regular path.  On Debian/Ubuntu
 platforms, you will need to have the "systemd-journal-remote" package

roles/process-stackviz/tasks/main.yaml

@@ -1,70 +1,73 @@
-- name: Devstack checks if stackviz archive exists
-  stat:
-    path: "/opt/cache/files/stackviz-latest.tar.gz"
-  register: stackviz_archive
-
-- debug:
-    msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz"
-  when: not stackviz_archive.stat.exists
-
-- name: Check if subunit data exists
-  stat:
-    path: "{{ zuul_work_dir }}/testrepository.subunit"
-  register: subunit_input
-
-- debug:
-    msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit"
-  when: not subunit_input.stat.exists
-
-- name: Install stackviz
-  when:
-    - stackviz_archive.stat.exists
-    - subunit_input.stat.exists
+- name: Process Stackviz
   block:
-    - include_role:
-        name: ensure-pip
 
-    - pip:
-        name: "file://{{ stackviz_archive.stat.path }}"
-        virtualenv: /tmp/stackviz
-        virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
-        extra_args: -U
+  - name: Devstack checks if stackviz archive exists
+    stat:
+      path: "/opt/cache/files/stackviz-latest.tar.gz"
+    register: stackviz_archive
 
-- name: Deploy stackviz static html+js
-  command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
-  when:
-    - stackviz_archive.stat.exists
-    - subunit_input.stat.exists
+  - debug:
+      msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz"
+    when: not stackviz_archive.stat.exists
 
-- name: Check if dstat data exists
-  stat:
-    path: "{{ devstack_base_dir }}/logs/dstat-csv.log"
-  register: dstat_input
-  when:
-    - stackviz_archive.stat.exists
-    - subunit_input.stat.exists
+  - name: Check if subunit data exists
+    stat:
+      path: "{{ zuul_work_dir }}/testrepository.subunit"
+    register: subunit_input
 
-- name: Run stackviz with dstat
-  shell: |
-    cat {{ subunit_input.stat.path }} | \
-      /tmp/stackviz/bin/stackviz-export \
-        --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \
-        --env --stdin \
-        {{ stage_dir }}/stackviz/data
-  when:
-    - stackviz_archive.stat.exists
-    - subunit_input.stat.exists
-    - dstat_input.stat.exists
-  failed_when: False
+  - debug:
+      msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit"
+    when: not subunit_input.stat.exists
 
-- name: Run stackviz without dstat
-  shell: |
-    cat {{ subunit_input.stat.path }} | \
-      /tmp/stackviz/bin/stackviz-export \
-        --env --stdin \
-        {{ stage_dir }}/stackviz/data
-  when:
-    - stackviz_archive.stat.exists
-    - subunit_input.stat.exists
-    - not dstat_input.stat.exists
-  failed_when: False
+  - name: Install stackviz
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+    block:
+      - include_role:
+          name: ensure-pip
+
+      - pip:
+          name: "file://{{ stackviz_archive.stat.path }}"
+          virtualenv: /tmp/stackviz
+          virtualenv_command: '{{ ensure_pip_virtualenv_command }}'
+          extra_args: -U
+
+  - name: Deploy stackviz static html+js
+    command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+
+  - name: Check if dstat data exists
+    stat:
+      path: "{{ devstack_base_dir }}/logs/dstat-csv.log"
+    register: dstat_input
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+
+  - name: Run stackviz with dstat
+    shell: |
+      cat {{ subunit_input.stat.path }} | \
+        /tmp/stackviz/bin/stackviz-export \
+          --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \
+          --env --stdin \
+          {{ stage_dir }}/stackviz/data
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+      - dstat_input.stat.exists
+
+  - name: Run stackviz without dstat
+    shell: |
+      cat {{ subunit_input.stat.path }} | \
+        /tmp/stackviz/bin/stackviz-export \
+          --env --stdin \
+          {{ stage_dir }}/stackviz/data
+    when:
+      - stackviz_archive.stat.exists
+      - subunit_input.stat.exists
+      - not dstat_input.stat.exists
+
+  ignore_errors: yes

stack.sh

@@ -227,7 +227,7 @@ write_devstack_version
 
 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``FORCE=yes ./stack``
-SUPPORTED_DISTROS="bionic|focal|f31|f32|opensuse-15.2|opensuse-tumbleweed|rhel8"
+SUPPORTED_DISTROS="bionic|focal|f31|f32|opensuse-15.2|opensuse-tumbleweed|rhel8|rhel9"
 
 if [[ ! ${DISTRO} =~ $SUPPORTED_DISTROS ]]; then
     echo "WARNING: this script has not been tested on $DISTRO"
@@ -300,10 +300,27 @@ function _install_epel {
 }
 
 function _install_rdo {
-    # NOTE(ianw) 2020-04-30 : when we have future branches, we
-    # probably want to install the relevant branch RDO release as
-    # well.  But for now it's all master.
-    sudo dnf -y install https://rdoproject.org/repos/rdo-release.el8.rpm
+    if [[ $DISTRO == "rhel8" ]]; then
+        if [[ "$TARGET_BRANCH" == "master" ]]; then
+            # rdo-release.el8.rpm points to latest RDO release, use that for master
+            sudo dnf -y install https://rdoproject.org/repos/rdo-release.el8.rpm
+        else
+            # For stable branches use corresponding release rpm
+            rdo_release=$(echo $TARGET_BRANCH | sed "s|stable/||g")
+            sudo dnf -y install https://rdoproject.org/repos/openstack-${rdo_release}/rdo-release-${rdo_release}.el8.rpm
+        fi
+    elif [[ $DISTRO == "rhel9" ]]; then
+        if [[ "$TARGET_BRANCH" == "master" ]]; then
+            # rdo-release.el9.rpm points to latest RDO release, use that for master
+            sudo dnf -y install https://rdoproject.org/repos/rdo-release.el9.rpm
+        else
+            # RDO didn't publish official releases for Xena and Wallaby, only RDO Trunk repos. 
+            # Since the RDO Trunk server does not support EMS, FIPS jobs will fail to use it.
+            # Use "rdo-release-yoga" release rpm for both xena and wallaby jobs.
+            rdo_release="yoga"
+            sudo dnf -y install https://rdoproject.org/repos/openstack-${rdo_release}/rdo-release-${rdo_release}.el9.rpm
+        fi
+    fi
     sudo dnf -y update
 }
 
@@ -381,6 +398,10 @@ if [[ $DISTRO == "rhel8" ]]; then
     # RHBZ: https://bugzilla.redhat.com/show_bug.cgi?id=1154272
     # Patch: https://github.com/rpm-software-management/dnf/pull/1448
     echo "[]" | sudo tee /var/cache/dnf/expired_repos.json
+elif [[ $DISTRO == "rhel9" ]]; then
+    sudo dnf config-manager --set-enabled crb
+    # rabbitmq and other packages are provided by RDO repositories.
+    _install_rdo
 fi
 
 # Ensure python is installed
@@ -1238,17 +1259,21 @@ fi
 # deployments.  This ensures the keys match across nova and cinder across all
 # hosts.
 FIXED_KEY=${FIXED_KEY:-bae3516cc1c0eb18b05440eba8012a4a880a2ee04d584a9c1579445e675b12defdc716ec}
-if is_service_enabled nova; then
-    iniset $NOVA_CONF key_manager fixed_key "$FIXED_KEY"
-    iniset $NOVA_CPU_CONF key_manager fixed_key "$FIXED_KEY"
-fi
-
 if is_service_enabled cinder; then
     iniset $CINDER_CONF key_manager fixed_key "$FIXED_KEY"
 fi
 
 async_wait configure_neutron_nova
 
+# NOTE(clarkb): This must come after async_wait configure_neutron_nova because
+# configure_neutron_nova modifies $NOVA_CONF and $NOVA_CPU_CONF as well. If
+# we don't wait then these two ini updates race either other and can result
+# in unexpected configs.
+if is_service_enabled nova; then
+    iniset $NOVA_CONF key_manager fixed_key "$FIXED_KEY"
+    iniset $NOVA_CPU_CONF key_manager fixed_key "$FIXED_KEY"
+fi
+
 # Launch the nova-api and wait for it to answer before continuing
 if is_service_enabled n-api; then
     echo_summary "Starting Nova API"

stackrc

@@ -13,15 +13,15 @@ RC_DIR=$(cd $(dirname "${BASH_SOURCE:-$0}") && pwd)
 # Source required DevStack functions and globals
 source $RC_DIR/functions
 
-# Set the target branch. This is used so that stable branching
+# Set the target branch. This is used so that unmaintained branching
 # does not need to update each repo below.
-TARGET_BRANCH=master
+TARGET_BRANCH=unmaintained/wallaby
 
 # Cycle trailing projects need to branch later than the others.
 TRAILING_TARGET_BRANCH=master
 
-# And some repos do not create stable branches, so this is used
-# to make it explicit and avoid accidentally setting to a stable
+# And some repos do not create unmaintained branches, so this is used
+# to make it explicit and avoid accidentally setting to a unmaintained
 # branch.
 BRANCHLESS_TARGET_BRANCH=master
 
@@ -203,6 +203,11 @@ ADDITIONAL_VENV_PACKAGES=${ADITIONAL_VENV_PACKAGES:-""}
 # (currently only implemented for MySQL backend)
 DATABASE_QUERY_LOGGING=$(trueorfalse False DATABASE_QUERY_LOGGING)
 
+# This can be used to reduce the amount of memory mysqld uses while running.
+# These are unscientifically determined, and could reduce performance or
+# cause other issues.
+MYSQL_REDUCE_MEMORY=$(trueorfalse False MYSQL_REDUCE_MEMORY)
+
 # Set a timeout for git operations.  If git is still running when the
 # timeout expires, the command will be retried up to 3 times.  This is
 # in the format for timeout(1);
@@ -297,8 +302,8 @@ REQUIREMENTS_BRANCH=${REQUIREMENTS_BRANCH:-$TARGET_BRANCH}
 
 # Tempest test suite
 TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git}
-TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH}
-TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master}
+TEMPEST_BRANCH=${TEMPEST_BRANCH:-29.0.0}
+TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-$REQUIREMENTS_DIR/upper-constraints.txt}
 
 
 ##############
@@ -758,8 +763,8 @@ for image_url in ${IMAGE_URLS//,/ }; do
     fi
 done
 
-# 24Gb default volume backing file size
-VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-24G}
+# 30Gb default volume backing file size
+VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-30G}
 
 # Prefixes for volume and instance names
 VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}

tests/test_libs_from_pypi.sh

@@ -95,19 +95,7 @@ function test_libs_exist {
     echo "test_libs_exist PASSED"
 }
 
-function test_branch_master {
-    for lib in $ALL_LIBS; do
-        if [[ ${GITBRANCH[$lib]} != "master" ]]; then
-            echo "GITBRANCH for $lib not master (${GITBRANCH[$lib]})"
-            exit 1
-        fi
-    done
-
-    echo "test_branch_master PASSED"
-}
-
 set -o errexit
 
 test_libs_exist
-test_branch_master
 test_all_libs_upto_date

tests/test_refs.sh

@@ -15,7 +15,7 @@
 
 echo "Ensuring we don't have crazy refs"
 
-REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH'`
+REFS=`grep BRANCH stackrc | grep -v 'TARGET_BRANCH' | grep -v 'NOVNC_BRANCH' | grep -v 'TEMPEST_BRANCH'`
 rc=$?
 if [[ $rc -eq 0 ]]; then
     echo "Branch defaults must be one of the *TARGET_BRANCH values. Found:"

tests/test_write_devstack_local_conf_role.sh

@@ -6,4 +6,4 @@ TOP=$(cd $(dirname "$0")/.. && pwd)
 source $TOP/functions
 source $TOP/tests/unittest.sh
 
-python ./roles/write-devstack-local-conf/library/test.py
+${PYTHON} $TOP/roles/write-devstack-local-conf/library/test.py

tests/unittest.sh

@@ -17,6 +17,8 @@ ERROR=0
 PASS=0
 FAILED_FUNCS=""
 
+export PYTHON=$(which python3 2>/dev/null)
+
 # pass a test, printing out MSG
 #  usage: passed message
 function passed {

tools/cap-pip.txt

@@ -1 +0,0 @@
-pip<20.3

tools/fixup_stuff.sh

@@ -192,6 +192,22 @@ function fixup_ovn_centos {
     yum_install centos-release-openstack-victoria
 }
 
+function fixup_ubuntu {
+    if ! is_ubuntu; then
+        return
+    fi
+
+    # Since pip10, pip will refuse to uninstall files from packages
+    # that were created with distutils (rather than more modern
+    # setuptools).  This is because it technically doesn't have a
+    # manifest of what to remove.  However, in most cases, simply
+    # overwriting works.  So this hacks around those packages that
+    # have been dragged in by some other system dependency
+    sudo rm -rf /usr/lib/python3/dist-packages/PyYAML-*.egg-info
+    sudo rm -rf /usr/lib/python3/dist-packages/pyasn1_modules-*.egg-info
+    sudo rm -rf /usr/lib/python3/dist-packages/simplejson-*.egg-info
+}
+
 function fixup_all {
     fixup_keystone
     fixup_ubuntu

tools/install_pip.sh

@@ -38,7 +38,7 @@ FILES=$TOP_DIR/files
 # [1] https://opendev.org/openstack/project-config/src/branch/master/nodepool/elements/cache-devstack/source-repository-pip
 
 PIP_GET_PIP_URL=${PIP_GET_PIP_URL:-"https://bootstrap.pypa.io/get-pip.py"}
-LOCAL_PIP="$FILES/$(basename $PIP_GET_PIP_URL)"
+PIP_GET_PIP36_URL=${PIP_GET_PIP36_URL:-"https://bootstrap.pypa.io/pip/3.6/get-pip.py"}
 
 GetDistro
 echo "Distro: $DISTRO"
@@ -59,12 +59,21 @@ function get_versions {
 
 
 function install_get_pip {
+    if [[ "$PYTHON3_VERSION" = "3.6" ]]; then
+        _pip_url=$PIP_GET_PIP36_URL
+        _local_pip="$FILES/$(basename $_pip_url)-py36"
+    else
+        _pip_url=$PIP_GET_PIP_URL
+        _local_pip="$FILES/$(basename $_pip_url)"
+    fi
+
+
     # If get-pip.py isn't python, delete it. This was probably an
     # outage on the server.
-    if [[ -r $LOCAL_PIP ]]; then
-        if ! head -1 $LOCAL_PIP | grep -q '#!/usr/bin/env python'; then
-            echo "WARNING: Corrupt $LOCAL_PIP found removing"
-            rm $LOCAL_PIP
+    if [[ -r $_local_pip ]]; then
+        if ! head -1 $_local_pip | grep -q '#!/usr/bin/env python'; then
+            echo "WARNING: Corrupt $_local_pip found removing"
+            rm $_local_pip
         fi
     fi
 
@@ -78,22 +87,20 @@ function install_get_pip {
     # Thus we use curl's "-z" feature to always check the modified
     # since and only download if a new version is out -- but only if
     # it seems we downloaded the file originally.
-    if [[ ! -r $LOCAL_PIP || -r $LOCAL_PIP.downloaded ]]; then
+    if [[ ! -r $_local_pip || -r $_local_pip.downloaded ]]; then
         # only test freshness if LOCAL_PIP is actually there,
         # otherwise we generate a scary warning.
         local timecond=""
-        if [[ -r $LOCAL_PIP ]]; then
-            timecond="-z $LOCAL_PIP"
+        if [[ -r $_local_pip ]]; then
+            timecond="-z $_local_pip"
         fi
 
         curl -f --retry 6 --retry-delay 5 \
-            $timecond -o $LOCAL_PIP $PIP_GET_PIP_URL || \
+            $timecond -o $_local_pip $_pip_url || \
             die $LINENO "Download of get-pip.py failed"
-        touch $LOCAL_PIP.downloaded
+        touch $_local_pip.downloaded
     fi
-    # TODO: remove the trailing pip constraint when a proper fix
-    # arrives for bug https://bugs.launchpad.net/devstack/+bug/1906322
-    sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
+    sudo -H -E python${PYTHON3_VERSION} $_local_pip
 }
 
 
@@ -130,28 +137,29 @@ if [[ -n $PYPI_ALTERNATIVE_URL ]]; then
     configure_pypi_alternative_url
 fi
 
-# Just use system pkgs on Focal
-if [[ "$DISTRO" == focal ]]; then
-    exit 0
-fi
-
-# Eradicate any and all system packages
-
-# Python in fedora/suse depends on the python-pip package so removing it
-# results in a nonfunctional system. pip on fedora installs to /usr so pip
-# can safely override the system pip for all versions of fedora
-if ! is_fedora  && ! is_suse; then
-    if is_package_installed python3-pip ; then
-        uninstall_package python3-pip
+if is_fedora && [[ ${DISTRO} == f* || ${DISTRO} == rhel9 ]]; then
+    # get-pip.py will not install over the python3-pip package in
+    # Fedora 34 any more.
+    #  https://bugzilla.redhat.com/show_bug.cgi?id=1988935
+    #  https://github.com/pypa/pip/issues/9904
+    # You can still install using get-pip.py if python3-pip is *not*
+    # installed; this *should* remain separate under /usr/local and not break
+    # if python3-pip is later installed.
+    # For general sanity, we just use the packaged pip.  It should be
+    # recent enough anyway.  This is included via rpms/general
+    : # Simply fall through
+else
+    if ! is_fedora && ! is_suse; then
+        if is_package_installed python3-pip ; then
+            uninstall_package python3-pip
+        fi
     fi
+    install_get_pip
+    # Note setuptools is part of requirements.txt and we want to make sure
+    # we obey any versioning as described there.
+    pip_install_gr setuptools
 fi
 
-install_get_pip
-
 set -x
 
-# Note setuptools is part of requirements.txt and we want to make sure
-# we obey any versioning as described there.
-pip_install_gr setuptools
-
 get_versions

tools/verify-ipv6-only-deployments.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+#
+#
+# NOTE(gmann): This script is used in 'devstack-tempest-ipv6' zuul job to verify that
+# services are deployed on IPv6 properly or not. This will capture if any devstck or devstack
+# plugins are missing the required setting to listen on IPv6 address. This is run as part of
+# run phase of zuul job and before test run. Child job of 'devstack-tempest-ipv6'
+# can expand the IPv6 verification specific to project by defining the new post-run script which
+# will run along with this base script.
+# If there are more common verification for IPv6 then we can always extent this script.
+
+# Keep track of the DevStack directory
+TOP_DIR=$(cd $(dirname "$0")/../../devstack && pwd)
+source $TOP_DIR/stackrc
+source $TOP_DIR/openrc admin admin
+
+function verify_devstack_ipv6_setting {
+    local _service_host=''
+    _service_host=$(echo $SERVICE_HOST | tr -d [])
+    local _host_ipv6=''
+    _host_ipv6=$(echo $HOST_IPV6 | tr -d [])
+    local _service_listen_address=''
+    _service_listen_address=$(echo $SERVICE_LISTEN_ADDRESS | tr -d [])
+    local _service_local_host=''
+    _service_local_host=$(echo $SERVICE_LOCAL_HOST | tr -d [])
+    if [[ "$SERVICE_IP_VERSION" != 6 ]]; then
+        echo $SERVICE_IP_VERSION "SERVICE_IP_VERSION is not set to 6 which is must for devstack to deploy services with IPv6 address."
+        exit 1
+    fi
+    is_service_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_host'"))')
+    if [[ "$is_service_host_ipv6" != "True" ]]; then
+        echo $SERVICE_HOST "SERVICE_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address."
+        exit 1
+    fi
+    is_host_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_host_ipv6'"))')
+    if [[ "$is_host_ipv6" != "True" ]]; then
+        echo $HOST_IPV6 "HOST_IPV6 is not ipv6 which means devstack cannot deploy services on IPv6 address."
+        exit 1
+    fi
+    is_service_listen_address=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_listen_address'"))')
+    if [[ "$is_service_listen_address" != "True" ]]; then
+        echo $SERVICE_LISTEN_ADDRESS "SERVICE_LISTEN_ADDRESS is not ipv6 which means devstack cannot deploy services on IPv6 address."
+        exit 1
+    fi
+    is_service_local_host=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$_service_local_host'"))')
+    if [[ "$is_service_local_host" != "True" ]]; then
+        echo $SERVICE_LOCAL_HOST "SERVICE_LOCAL_HOST is not ipv6 which means devstack cannot deploy services on IPv6 address."
+        exit 1
+    fi
+    echo "Devstack is properly configured with IPv6"
+    echo "SERVICE_IP_VERSION: " $SERVICE_IP_VERSION "HOST_IPV6: " $HOST_IPV6 "SERVICE_HOST: " $SERVICE_HOST "SERVICE_LISTEN_ADDRESS: " $SERVICE_LISTEN_ADDRESS "SERVICE_LOCAL_HOST: " $SERVICE_LOCAL_HOST
+}
+
+function sanity_check_system_ipv6_enabled {
+    system_ipv6_enabled=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_ipv6_enabled())')
+    if [[ $system_ipv6_enabled != "True" ]]; then
+        echo "IPv6 is disabled in system"
+        exit 1
+    fi
+    echo "IPv6 is enabled in system"
+}
+
+function verify_service_listen_address_is_ipv6 {
+    local endpoints_verified=False
+    local all_ipv6=True
+    endpoints=$(openstack endpoint list -f value -c URL)
+    for endpoint in ${endpoints}; do
+        local endpoint_address=''
+        endpoint_address=$(echo "$endpoint" | awk -F/ '{print $3}' | awk -F] '{print $1}')
+        endpoint_address=$(echo $endpoint_address | tr -d [])
+        local is_endpoint_ipv6=''
+        is_endpoint_ipv6=$(python3 -c 'import oslo_utils.netutils as nutils; print(nutils.is_valid_ipv6("'$endpoint_address'"))')
+        if [[ "$is_endpoint_ipv6" != "True" ]]; then
+            all_ipv6=False
+            echo $endpoint ": This is not ipv6 endpoint which means corresponding service is not listening on IPv6 address."
+            continue
+        fi
+        endpoints_verified=True
+    done
+    if [[ "$all_ipv6" == "False"  ]] || [[ "$endpoints_verified" == "False" ]]; then
+        exit 1
+    fi
+    echo "All services deployed by devstack is on IPv6 endpoints"
+    echo $endpoints
+}
+
+#First thing to verify if system has IPv6 enabled or not
+sanity_check_system_ipv6_enabled
+#Verify whether devstack is configured properly with IPv6 setting
+verify_devstack_ipv6_setting
+#Get all registrfed endpoints by devstack in keystone and verify that each endpoints address is IPv6.
+verify_service_listen_address_is_ipv6

tools/worlddump.py

@@ -134,7 +134,7 @@ def disk_space():
 
 
 def ebtables_dump():
-    tables = ['filter', 'nat', 'broute']
+    tables = ['filter', 'nat']
     _header("EB Tables Dump")
     if not _find_cmd('ebtables'):
         return

tox.ini

@@ -34,7 +34,9 @@ commands = bash -c "find {toxinidir}             \
          -print0 | xargs -0 bashate -v -iE006 -eE005,E042"
 
 [testenv:docs]
-deps = -r{toxinidir}/doc/requirements.txt
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/wallaby}
+  -r{toxinidir}/doc/requirements.txt
 whitelist_externals = bash
 setenv =
   TOP_DIR={toxinidir}

unstack.sh

@@ -185,3 +185,8 @@ fi
 
 clean_pyc_files
 rm -Rf $DEST/async
+
+# Clean any safe.directory items we wrote into the global
+# gitconfig. We can identify the relevant ones by checking that they
+# point to somewhere in our $DEST directory.
+sudo sed -i "/directory=${DEST}/ d" /etc/gitconfig