Merge "Replace deprecated datetime.utcnow()"

Merge "Globally skip devstack job for pre-commit config update"
Merge "Add image format enforcement toggle"
2024-10-25 08:38:40 +00:00 · 2024-10-25 07:56:22 +00:00 · 2024-10-25 07:56:19 +00:00 · 2024-10-24 00:54:28 +09:00 · 2024-10-21 14:22:36 +00:00 · 2024-10-19 12:55:43 +00:00
186 changed files with 6302 additions and 4104 deletions
@@ -38,3 +38,5 @@ stack-screenrc
 userrc_early
 AUTHORS
 ChangeLog
+tools/dbcounter/build/
+tools/dbcounter/dbcounter.egg-info/
@@ -1,8 +1,18 @@
 - nodeset:
-    name: openstack-single-node
+    name: openstack-single-node-jammy
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: ubuntu-jammy
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: openstack-single-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
    groups:
      - name: tempest
        nodes:
@@ -29,30 +39,10 @@
          - controller

 - nodeset:
-    name: openstack-single-node-xenial
+    name: devstack-single-node-centos-9-stream
    nodes:
      - name: controller
-        label: ubuntu-xenial
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-7
-    nodes:
-      - name: controller
-        label: centos-7
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-8
-    nodes:
-      - name: controller
-        label: centos-8
+        label: centos-9-stream
    groups:
      - name: tempest
        nodes:
@@ -69,22 +59,105 @@
          - controller

 - nodeset:
-    name: devstack-single-node-fedora-latest
+    name: devstack-single-node-debian-bookworm
    nodes:
      - name: controller
-        label: fedora-32
+        label: debian-bookworm
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+# Note(sean-k-mooney): this is still used by horizon for
+# horizon-integration-tests, horizon-integration-pytest and
+# horizon-ui-pytest, remove when horizon is updated.
+- nodeset:
+    name: devstack-single-node-debian-bullseye
+    nodes:
+      - name: controller
+        label: debian-bullseye
    groups:
      - name: tempest
        nodes:
          - controller

 - nodeset:
-    name: openstack-two-node
+    name: devstack-single-node-rockylinux-9
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: rockylinux-9
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: openstack-two-node-centos-9-stream
+    nodes:
+      - name: controller
+        label: centos-9-stream
      - name: compute1
-        label: ubuntu-xenial
+        label: centos-9-stream
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
+- nodeset:
+    name: openstack-two-node-jammy
+    nodes:
+      - name: controller
+        label: ubuntu-jammy
+      - name: compute1
+        label: ubuntu-jammy
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
+- nodeset:
+    name: openstack-two-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
+      - name: compute1
+        label: ubuntu-noble
    groups:
      # Node where tests are executed and test results collected
      - name: tempest
@@ -168,36 +241,6 @@
        nodes:
          - compute1

- nodeset:
-    name: openstack-two-node-xenial
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-      - name: compute1
-        label: ubuntu-xenial
-    groups:
-      # Node where tests are executed and test results collected
-      - name: tempest
-        nodes:
-          - controller
-      # Nodes running the compute service
-      - name: compute
-        nodes:
-          - controller
-          - compute1
-      # Nodes that are not the controller
-      - name: subnode
-        nodes:
-          - compute1
-      # Switch node for multinode networking setup
-      - name: switch
-        nodes:
-          - controller
-      # Peer nodes for multinode networking setup
-      - name: peers
-        nodes:
-          - compute1
-
 - nodeset:
    name: openstack-three-node-focal
    nodes:
@@ -270,7 +313,7 @@

 - job:
    name: devstack-base
-    parent: multinode
+    parent: openstack-multinode-fips
    abstract: true
    description: |
      Base abstract Devstack job.
@@ -285,7 +328,6 @@
    required-projects:
      - opendev.org/openstack/devstack
    roles:
-      - zuul: opendev.org/openstack/devstack-gate
      - zuul: opendev.org/openstack/openstack-zuul-jobs
    vars:
      devstack_localrc:
@@ -319,8 +361,10 @@
        '{{ devstack_log_dir }}/devstacklog.txt.summary': logs
        '{{ devstack_log_dir }}/tcpdump.pcap': logs
        '{{ devstack_log_dir }}/worlddump-latest.txt': logs
+        '{{ devstack_log_dir }}/qemu.coredump': logs
        '{{ devstack_full_log}}': logs
        '{{ stage_dir }}/verify_tempest_conf.log': logs
+        '{{ stage_dir }}/performance.json': logs
        '{{ stage_dir }}/apache': logs
        '{{ stage_dir }}/apache_config': logs
        '{{ stage_dir }}/etc': logs
@@ -328,6 +372,8 @@
        /var/log/postgresql: logs
        /var/log/mysql: logs
        /var/log/libvirt: logs
+        /etc/libvirt: logs
+        /etc/lvm: logs
        /etc/sudoers: logs
        /etc/sudoers.d: logs
        '{{ stage_dir }}/iptables.txt': logs
@@ -338,8 +384,10 @@
        '{{ stage_dir }}/rpm-qa.txt': logs
        '{{ stage_dir }}/core': logs
        '{{ stage_dir }}/listen53.txt': logs
+        '{{ stage_dir }}/services.txt': logs
        '{{ stage_dir }}/deprecations.log': logs
        '{{ stage_dir }}/audit.log': logs
+        /etc/ceph: logs
        /var/log/ceph: logs
        /var/log/openvswitch: logs
        /var/log/glusterfs: logs
@@ -384,6 +432,8 @@
      - ^releasenotes/.*$
      # Translations
      - ^.*/locale/.*po$
+      # pre-commit config
+      - ^.pre-commit-config.yaml$

 - job:
    name: devstack-minimal
@@ -391,7 +441,7 @@
    description: |
      Minimal devstack base job, intended for use by jobs that need
      less than the normal minimum set of required-projects.
-    nodeset: openstack-single-node-focal
+    nodeset: openstack-single-node-jammy
    required-projects:
      - opendev.org/openstack/requirements
    vars:
@@ -402,17 +452,21 @@
        PUBLIC_BRIDGE_MTU: '{{ external_bridge_mtu }}'
      devstack_services:
        # Shared services
-        dstat: true
+        dstat: false
        etcd3: true
        memory_tracker: true
+        file_tracker: true
        mysql: true
        rabbit: true
+        openstack-cli-server: true
    group-vars:
      subnode:
        devstack_services:
          # Shared services
-          dstat: true
+          dstat: false
          memory_tracker: true
+          file_tracker: true
+          openstack-cli-server: true
        devstack_localrc:
          # Multinode specific settings
          HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}"
@@ -458,8 +512,17 @@
      - opendev.org/openstack/nova
      - opendev.org/openstack/placement
      - opendev.org/openstack/swift
+      - opendev.org/openstack/os-test-images
    timeout: 7200
    vars:
+      # based on observation of the integrated gate
+      # tempest-integrated-compute was only using ~1.7GB of swap
+      # when zswap and the host turning are enabled that increase
+      # slightly to ~2GB. we are setting the swap size to 8GB to
+      # be safe and account for more complex scenarios.
+      # we should revisit this value after some time to see if we
+      # can reduce it.
+      configure_swap_size: 8192
      devstack_localrc:
        # Common OpenStack services settings
        SWIFT_REPLICAS: 1
@@ -467,6 +530,27 @@
        SWIFT_HASH: 1234123412341234
        DEBUG_LIBVIRT_COREDUMPS: true
        NOVA_VNC_ENABLED: true
+        OVN_DBS_LOG_LEVEL: dbg
+        # tune the host to optimize memory usage and hide io latency
+        # these setting will configure the kernel to treat the host page
+        # cache and swap with equal priority, and prefer deferring writes
+        # changing the default swappiness, dirty_ratio and
+        # the vfs_cache_pressure
+        ENABLE_SYSCTL_MEM_TUNING: true
+        # the net tuning optimizes ipv4 tcp fast open and config the default
+        # qdisk policy to pfifo_fast which effectively disable all qos.
+        # this minimizes the cpu load of the host network stack
+        ENABLE_SYSCTL_NET_TUNING: true
+        # zswap allows the kernel to compress pages in memory before swapping
+        # them to disk. this can reduce the amount of swap used and improve
+        # performance. effectively this trades a small amount of cpu for an
+        # increase in swap performance by reducing the amount of data
+        # written to disk. the overall speedup is proportional to the
+        # compression ratio and the speed of the swap device.
+        # NOTE: this option is ignored when not using nova with the libvirt
+        # virt driver.
+        NOVA_LIBVIRT_TB_CACHE_SIZE: 128
+        ENABLE_ZSWAP: true
      devstack_local_conf:
        post-config:
          $NEUTRON_CONF:
@@ -476,9 +560,10 @@
        # Core services enabled for this branch.
        # This list replaces the test-matrix.
        # Shared services
-        dstat: true
+        dstat: false
        etcd3: true
        memory_tracker: true
+        file_tracker: true
        mysql: true
        rabbit: true
        tls-proxy: true
@@ -495,13 +580,14 @@
        n-sch: true
        # Placement service
        placement-api: true
+        # OVN services
+        ovn-controller: true
+        ovn-northd: true
+        ovs-vswitchd: true
+        ovsdb-server: true
        # Neutron services
-        q-agt: true
-        q-dhcp: true
-        q-l3: true
-        q-meta: true
-        q-metering: true
        q-svc: true
+        q-ovn-metadata-agent: true
        # Swift services
        s-account: true
        s-container: true
@@ -512,7 +598,6 @@
        c-bak: true
        c-sch: true
        c-vol: true
-        cinder: true
        # Services we don't need.
        # This section is not really needed, it's for readability.
        horizon: false
@@ -526,15 +611,20 @@
          # Core services enabled for this branch.
          # This list replaces the test-matrix.
          # Shared services
-          dstat: true
+          dstat: false
          memory_tracker: true
+          file_tracker: true
          tls-proxy: true
          # Nova services
          n-cpu: true
          # Placement services
          placement-client: true
+          # OVN services
+          ovn-controller: true
+          ovs-vswitchd: true
+          ovsdb-server: true
          # Neutron services
-          q-agt: true
+          q-ovn-metadata-agent: true
          # Cinder services
          c-bak: true
          c-vol: true
@@ -552,21 +642,53 @@
          GLANCE_HOSTPORT: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}:9292"
          Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
          NOVA_VNC_ENABLED: true
+          ENABLE_CHASSIS_AS_GW: false
+          # tune the host to optimize memory usage and hide io latency
+          # these setting will configure the kernel to treat the host page
+          # cache and swap with equal priority, and prefer deferring writes
+          # changing the default swappiness, dirty_ratio and
+          # the vfs_cache_pressure
+          ENABLE_SYSCTL_MEM_TUNING: true
+          # the net tuning optimizes ipv4 tcp fast open and config the default
+          # qdisk policy to pfifo_fast which effectively disable all qos.
+          # this minimizes the cpu load of the host network stack
+          ENABLE_SYSCTL_NET_TUNING: true
+          # zswap allows the kernel to compress pages in memory before swapping
+          # them to disk. this can reduce the amount of swap used and improve
+          # performance. effectivly this trades a small amount of cpu for an
+          # increase in swap performance by reducing the amount of data
+          # written to disk. the overall speedup is porportional to the
+          # compression ratio and the speed of the swap device.
+          ENABLE_ZSWAP: true
+          # NOTE: this option is ignored when not using nova with the libvirt
+          # virt driver.
+          NOVA_LIBVIRT_TB_CACHE_SIZE: 128

 - job:
    name: devstack-ipv6
    parent: devstack
    description: |
-      Devstack single node job for integration gate with IPv6.
+      Devstack single node job for integration gate with IPv6,
+      all services and tunnels using IPv6 addresses.
    vars:
      devstack_localrc:
        SERVICE_IP_VERSION: 6
        SERVICE_HOST: ""
+        TUNNEL_IP_VERSION: 6
+
+- job:
+    name: devstack-enforce-scope
+    parent: devstack
+    description: |
+      This job runs the devstack with scope checks enabled.
+    vars:
+      devstack_localrc:
+        ENFORCE_SCOPE: true

 - job:
    name: devstack-multinode
    parent: devstack
-    nodeset: openstack-two-node-focal
+    nodeset: openstack-two-node-jammy
    description: |
      Simple multinode test to verify multinode functionality on devstack side.
      This is not meant to be used as a parent job.
@@ -576,43 +698,106 @@
 # and these platforms don't have the round-the-clock support to avoid
 # becoming blockers in that situation.
 - job:
-    name: devstack-platform-centos-8
+    name: devstack-platform-centos-9-stream
    parent: tempest-full-py3
-    description: Centos 8 platform test
-    nodeset: devstack-single-node-centos-8
-    voting: false
+    description: CentOS 9 Stream platform test
+    nodeset: devstack-single-node-centos-9-stream
    timeout: 9000
-
- job:
-    name: devstack-platform-opensuse-15
-    parent: tempest-full-py3
-    description: openSUSE 15.x platform test
-    nodeset: devstack-single-node-opensuse-15
    voting: false

 - job:
-    name: devstack-platform-bionic
+    name: devstack-platform-debian-bookworm
    parent: tempest-full-py3
-    description: Ubuntu Bionic platform test
-    nodeset: openstack-single-node-bionic
-    voting: false
+    description: Debian Bookworm platform test
+    nodeset: devstack-single-node-debian-bookworm
+    timeout: 9000
+    vars:
+      configure_swap_size: 4096

 - job:
-    name: devstack-platform-fedora-latest
+    name: devstack-platform-rocky-blue-onyx
    parent: tempest-full-py3
-    description: Fedora latest platform test
-    nodeset: devstack-single-node-fedora-latest
+    description: Rocky Linux 9 Blue Onyx platform test
+    nodeset: devstack-single-node-rockylinux-9
+    timeout: 9000
+    # NOTE(danms): This has been failing lately with some repository metadata
+    # errors. We're marking this as non-voting until it appears to have
+    # stabilized:
+    # https://zuul.openstack.org/builds?job_name=devstack-platform-rocky-blue-onyx&skip=0
    voting: false
+    vars:
+      configure_swap_size: 4096

 - job:
-    name: devstack-platform-fedora-latest-virt-preview
+    name: devstack-platform-ubuntu-noble
    parent: tempest-full-py3
-    description: Fedora latest platform test using the virt-preview repo.
-    nodeset: devstack-single-node-fedora-latest
+    description: Ubuntu 24.04 LTS (noble) platform test
+    nodeset: openstack-single-node-noble
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovn-source
+    parent: devstack-platform-ubuntu-jammy
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVN from source)
    voting: false
    vars:
      devstack_localrc:
-        ENABLE_FEDORA_VIRT_PREVIEW_REPO: true
+        OVN_BUILD_FROM_SOURCE: True
+        OVN_BRANCH: "v21.06.0"
+        OVS_BRANCH: "a4b04276ab5934d087669ff2d191a23931335c87"
+        OVS_SYSCONFDIR: "/usr/local/etc/openvswitch"
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovs
+    parent: tempest-full-py3
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVS)
+    nodeset: openstack-single-node-jammy
+    voting: false
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+      devstack_localrc:
+        Q_AGENT: openvswitch
+        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
+        Q_ML2_TENANT_NETWORK_TYPE: vxlan
+      devstack_services:
+        # Disable OVN services
+        ovn-northd: false
+        ovn-controller: false
+        ovs-vswitchd: false
+        ovsdb-server: false
+        # Disable Neutron ML2/OVN services
+        q-ovn-metadata-agent: false
+        # Enable Neutron ML2/OVS services
+        q-agt: true
+        q-dhcp: true
+        q-l3: true
+        q-meta: true
+        q-metering: true
+    group-vars:
+      subnode:
+        devstack_services:
+          # Disable OVN services
+          ovn-controller: false
+          ovs-vswitchd: false
+          ovsdb-server: false
+          # Disable Neutron ML2/OVN services
+          q-ovn-metadata-agent: false
+          # Enable Neutron ML2/OVS services
+          q-agt: true
+
+- job:
+    name: devstack-no-tls-proxy
+    parent: tempest-full-py3
+    description: |
+      Tempest job with tls-proxy off.
+
+      Some gates run devstack like this and it follows different code paths.
+    vars:
+      devstack_services:
+        tls-proxy: false

 - job:
    name: devstack-tox-base
@@ -670,6 +855,7 @@

 - job:
    name: devstack-unit-tests
+    nodeset: ubuntu-jammy
    description: |
      Runs unit tests on devstack project.

@@ -685,32 +871,23 @@
      jobs:
        - devstack
        - devstack-ipv6
-        - devstack-platform-opensuse-15
-        - devstack-platform-fedora-latest
-        - devstack-platform-centos-8
-        - devstack-platform-bionic
+        - devstack-enforce-scope
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
        - devstack-multinode
        - devstack-unit-tests
        - openstack-tox-bashate
-        - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa:
-            voting: false
-        - swift-dsvm-functional:
-            voting: false
-            irrelevant-files: &dsvm-irrelevant-files
-              - ^.*\.rst$
-              - ^doc/.*$
-        - swift-dsvm-functional-py3:
-            voting: false
-            irrelevant-files: *dsvm-irrelevant-files
+        - ironic-tempest-bios-ipmi-direct-tinyipa
+        - swift-dsvm-functional
        - grenade:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-grenade-multinode:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - neutron-tempest-linuxbridge:
+        - neutron-ovs-grenade-multinode:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@@ -736,30 +913,25 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        # NOTE(gmann): Remove this job from devstack pipeline once it is
-        # migrated to zuulv3 native. This is legacy job and rely on
-        # devstack-gate + devstack setting so any change in devstack can
-        # break it.
-        - nova-live-migration:
-            voting: false
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
    gate:
      jobs:
        - devstack
        - devstack-ipv6
+        - devstack-platform-debian-bookworm
+        - devstack-platform-ubuntu-noble
+        # NOTE(danms): Disabled due to instability, see comment in the job
+        # definition above.
+        # - devstack-platform-rocky-blue-onyx
+        - devstack-enforce-scope
        - devstack-multinode
        - devstack-unit-tests
        - openstack-tox-bashate
-        - neutron-grenade-multinode:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - neutron-tempest-linuxbridge:
+        - neutron-ovs-grenade-multinode:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
+        - ironic-tempest-bios-ipmi-direct-tinyipa
+        - swift-dsvm-functional
        - grenade:
            irrelevant-files:
              - ^.*\.rst$
@@ -781,7 +953,9 @@
    # pruned.
    #
    # * nova-next: maintained by nova for unreleased/undefaulted
-    #    things
+    #    things, this job is not experimental but often is used to test
+    #    things that are not yet production ready or to test what will be
+    #    the new default after a deprecation period has ended.
    # * neutron-fullstack-with-uwsgi: maintained by neutron for fullstack test
    #    when neutron-api is served by uwsgi, it's in exprimental for testing.
    #    the next cycle we can remove this  job if things turn out to be
@@ -789,14 +963,10 @@
    # * neutron-functional-with-uwsgi: maintained by neutron for functional
    #    test. Next cycle we can remove this one if things turn out to be
    #    stable engouh with uwsgi.
-    # * neutron-tempest-with-uwsgi: maintained by neutron for tempest test.
+    # * neutron-ovn-tempest-with-uwsgi: maintained by neutron for tempest test.
    #    Next cycle we can remove this if everything run out stable enough.
-    # * nova-multi-cell: maintained by nova and currently non-voting in the
+    # * nova-multi-cell: maintained by nova and now is voting in the
    #    check queue for nova changes but relies on devstack configuration
-    # * devstack-platform-fedora-latest-virt-preview: Maintained by lyarwood
-    #    for Nova to allow early testing of the latest versions of Libvirt and
-    #    QEMU. Should only graduate out of experimental if it ever moves into
-    #    the check queue for Nova.

    experimental:
      jobs:
@@ -804,16 +974,16 @@
        - nova-next
        - neutron-fullstack-with-uwsgi
        - neutron-functional-with-uwsgi
-        - neutron-tempest-with-uwsgi
+        - neutron-ovn-tempest-with-uwsgi
        - devstack-plugin-ceph-tempest-py3:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-tempest-dvr:
+        - neutron-ovs-tempest-dvr:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-tempest-dvr-ha-multinode-full:
+        - neutron-ovs-tempest-dvr-ha-multinode-full:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@@ -825,4 +995,15 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - devstack-platform-fedora-latest-virt-preview
+        - devstack-no-tls-proxy
+    periodic:
+      jobs:
+        - devstack-no-tls-proxy
+    periodic-weekly:
+      jobs:
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
@@ -74,8 +74,7 @@ of test of specific fragile functions in the ``functions`` and

 ``tools`` - Contains a collection of stand-alone scripts. While these
 may reference the top-level DevStack configuration they can generally be
-run alone. There are also some sub-directories to support specific
-environments such as XenServer.
+run alone.


 Scripts
@@ -275,9 +274,6 @@ your change
  even years from now -- why we were motivated to make a change at the
  time.

-* **Reviewers** -- please see ``MAINTAINERS.rst`` for a list of people
-  that should be added to reviews of various sub-systems.
-

 Making Changes, Testing, and CI
 -------------------------------
@@ -1,92 +0,0 @@
-MAINTAINERS
-===========
-
-
-Overview
--------
-
-The following is a list of people known to have interests in
-particular areas or sub-systems of devstack.
-
-It is a rather general guide intended to help seed the initial
-reviewers list of a change.  A +1 on a review from someone identified
-as being a maintainer of its affected area is a very positive flag to
-the core team for the veracity of the change.
-
-The ``devstack-core`` group can still be added to all reviews.
-
-
-Format
-~~~~~~
-
-The format of the file is the name of the maintainer and their
-gerrit-registered email.
-
-
-Maintainers
-----------
-
-.. contents:: :local:
-
-
-Ceph
-~~~~
-
-* Sebastien Han <sebastien.han@enovance.com>
-
-Cinder
-~~~~~~
-
-Fedora/CentOS/RHEL
-~~~~~~~~~~~~~~~~~~
-
-* Ian Wienand <iwienand@redhat.com>
-
-Neutron
-~~~~~~~
-
-MidoNet
-~~~~~~~
-
-* Jaume Devesa <devvesa@gmail.com>
-* Ryu Ishimoto <ryu@midokura.com>
-* YAMAMOTO Takashi <yamamoto@midokura.com>
-
-OpenDaylight
-~~~~~~~~~~~~
-
-* Kyle Mestery <mestery@mestery.com>
-
-OpenFlow Agent (ofagent)
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-* YAMAMOTO Takashi <yamamoto@valinux.co.jp>
-* Fumihiko Kakuma <kakuma@valinux.co.jp>
-
-Swift
-~~~~~
-
-* Chmouel Boudjnah <chmouel@enovance.com>
-
-SUSE
-~~~~
-
-* Ralf Haferkamp <rhafer@suse.de>
-* Vincent Untz <vuntz@suse.com>
-
-Tempest
-~~~~~~~
-
-Xen
-~~~
-* Bob Ball <bob.ball@citrix.com>
-
-Zaqar (Marconi)
-~~~~~~~~~~~~~~~
-
-* Flavio Percoco <flaper87@gmail.com>
-* Malini Kamalambal <malini.kamalambal@rackspace.com>
-
-Oracle Linux
-~~~~~~~~~~~~
-* Wiekus Beukes <wiekus.beukes@oracle.com>
@@ -4,7 +4,7 @@ from git source trees.
 Goals
 =====

-* To quickly build dev OpenStack environments in a clean Ubuntu or Fedora
+* To quickly build dev OpenStack environments in a clean Ubuntu or RockyLinux
  environment
 * To describe working configurations of OpenStack (which code branches
  work together?  what do config files look like for those branches?)
@@ -28,9 +28,9 @@ Versions
 The DevStack master branch generally points to trunk versions of OpenStack
 components.  For older, stable versions, look for branches named
 stable/[release] in the DevStack repo.  For example, you can do the
-following to create a Pike OpenStack cloud::
+following to create a Zed OpenStack cloud::

-    git checkout stable/pike
+    git checkout stable/zed
    ./stack.sh

 You can also pick specific OpenStack project releases by setting the appropriate
@@ -55,7 +55,7 @@ When the script finishes executing, you should be able to access OpenStack
 endpoints, like so:

 * Horizon: http://myhost/
-* Keystone: http://myhost/identity/v2.0/
+* Keystone: http://myhost/identity/v3/

 We also provide an environment file that you can use to interact with your
 cloud via CLI::
@@ -50,7 +50,6 @@ source $TOP_DIR/lib/placement
 source $TOP_DIR/lib/cinder
 source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/neutron
-source $TOP_DIR/lib/neutron-legacy

 set -o xtrace

@@ -113,7 +112,7 @@ cleanup_rpc_backend
 cleanup_database

 # Clean out data and status
-sudo rm -rf $DATA_DIR $DEST/status
+sudo rm -rf $DATA_DIR $DEST/status $DEST/async

 # Clean out the log file and log directories
 if [[ -n "$LOGFILE" ]] && [[ -f "$LOGFILE" ]]; then
@@ -145,3 +144,5 @@ done

 rm -rf ~/.config/openstack

+# Clear any fstab entries made
+sudo sed -i '/.*comment=devstack-.*/ d' /etc/fstab
@@ -4,8 +4,4 @@ Pygments
 docutils
 sphinx>=2.0.0,!=2.1.0 # BSD
 openstackdocstheme>=2.2.1 # Apache-2.0
-nwdiag
-blockdiag
-sphinxcontrib-blockdiag
-sphinxcontrib-nwdiag
 zuul-sphinx>=0.2.0
@@ -23,14 +23,14 @@

 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinx.ext.autodoc',
-               'zuul_sphinx',
-               'openstackdocstheme',
-               'sphinxcontrib.blockdiag',
-               'sphinxcontrib.nwdiag' ]
+extensions = [
+    'sphinx.ext.autodoc',
+    'zuul_sphinx',
+    'openstackdocstheme',
+]

 # openstackdocstheme options
-openstackdocs_repo_name = 'openstack-dev/devstack'
+openstackdocs_repo_name = 'openstack/devstack'
 openstackdocs_pdf_link = True
 openstackdocs_bug_project = 'devstack'
 openstackdocs_bug_tag = ''
@@ -181,6 +181,9 @@ values that most often need to be set.
 If the ``*_PASSWORD`` variables are not set here you will be prompted to
 enter values for them by ``stack.sh``.

+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.
+
 The network ranges must not overlap with any networks in use on the
 host. Overlap is not uncommon as RFC-1918 'private' ranges are commonly
 used for both the local networking and Nova's fixed and floating ranges.
@@ -279,7 +282,7 @@ number of days of old log files to keep.

 ::

-    LOGDAYS=1
+    LOGDAYS=2

 Some coloring is used during the DevStack runs to make it easier to
 see what is going on. This can be disabled with::
@@ -521,8 +524,8 @@ behavior:
 can be configured with any valid IPv6 prefix. The default values make
 use of an auto-generated ``IPV6_GLOBAL_ID`` to comply with RFC4193.

-Service Version
-~~~~~~~~~~~~~~~
+Service IP Version
+~~~~~~~~~~~~~~~~~~

 DevStack can enable service operation over either IPv4 or IPv6 by
 setting ``SERVICE_IP_VERSION`` to either ``SERVICE_IP_VERSION=4`` or
@@ -542,6 +545,27 @@ optionally be used to alter the default IPv6 address::

  HOST_IPV6=${some_local_ipv6_address}

+Tunnel IP Version
+~~~~~~~~~~~~~~~~~
+
+DevStack can enable tunnel operation over either IPv4 or IPv6 by
+setting ``TUNNEL_IP_VERSION`` to either ``TUNNEL_IP_VERSION=4`` or
+``TUNNEL_IP_VERSION=6`` respectively.
+
+When set to ``4`` Neutron will use an IPv4 address for tunnel endpoints,
+for example, ``HOST_IP``.
+
+When set to ``6`` Neutron will use an IPv6 address for tunnel endpoints,
+for example, ``HOST_IPV6``.
+
+The default value for this setting is ``4``.  Dual-mode support, for
+example ``4+6`` is not supported, as this value must match the address
+family of the local tunnel endpoint IP(v6) address.
+
+The value of ``TUNNEL_IP_VERSION`` has a direct relationship to the
+setting of ``TUNNEL_ENDPOINT_IP``, which will default to ``HOST_IP``
+when set to ``4``, and ``HOST_IPV6`` when set to ``6``.
+
 Multi-node setup
 ~~~~~~~~~~~~~~~~

@@ -615,7 +639,7 @@ tests can be run as follows:
 ::

    $ cd /opt/stack/tempest
-    $ tox -efull  tempest.scenario.test_network_basic_ops
+    $ tox -e smoke

 By default tempest is downloaded and the config file is generated, but the
 tempest package is not installed in the system's global site-packages (the
@@ -628,12 +652,6 @@ outside of tox. If you would like to install it add the following to your
    INSTALL_TEMPEST=True


-Xenserver
-~~~~~~~~~
-
-If you would like to use Xenserver as the hypervisor, please refer to
-the instructions in ``./tools/xen/README.md``.
-
 Cinder
 ~~~~~~

@@ -648,6 +666,41 @@ with ``VOLUME_BACKING_FILE_SIZE``.
  VOLUME_NAME_PREFIX="volume-"
  VOLUME_BACKING_FILE_SIZE=24G

+When running highly concurrent tests, the default per-project quotas
+for volumes, backups, or snapshots may be too small.  These can be
+adjusted by setting ``CINDER_QUOTA_VOLUMES``, ``CINDER_QUOTA_BACKUPS``,
+or ``CINDER_QUOTA_SNAPSHOTS`` to the desired value.  (The default for
+each is 10.)
+
+DevStack's Cinder LVM configuration module currently supports both iSCSI and
+NVMe connections, and we can choose which one to use with options
+``CINDER_TARGET_HELPER``, ``CINDER_TARGET_PROTOCOL``, ``CINDER_TARGET_PREFIX``,
+and ``CINDER_TARGET_PORT``.
+
+Defaults use iSCSI with the LIO target manager::
+
+  CINDER_TARGET_HELPER="lioadm"
+  CINDER_TARGET_PROTOCOL="iscsi"
+  CINDER_TARGET_PREFIX="iqn.2010-10.org.openstack:"
+  CINDER_TARGET_PORT=3260
+
+Additionally there are 3 supported transport protocols for NVMe,
+``nvmet_rdma``, ``nvmet_tcp``, and ``nvmet_fc``, and when the ``nvmet`` target
+is selected the protocol, prefix, and port defaults will change to more
+sensible defaults for NVMe::
+
+  CINDER_TARGET_HELPER="nvmet"
+  CINDER_TARGET_PROTOCOL="nvmet_rdma"
+  CINDER_TARGET_PREFIX="nvme-subsystem-1"
+  CINDER_TARGET_PORT=4420
+
+When selecting the RDMA transport protocol DevStack will create on Cinder nodes
+a Software RoCE device on top of the ``HOST_IP_IFACE`` and if it is not defined
+then on top of the interface with IP address ``HOST_IP`` or ``HOST_IPV6``.
+
+This Soft-RoCE device will always be created on the Nova compute side since we
+cannot tell beforehand whether there will be an RDMA connection or not.
+

 Keystone
 ~~~~~~~~
@@ -672,7 +725,6 @@ In RegionTwo:

    disable_service horizon
    KEYSTONE_SERVICE_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
-    KEYSTONE_AUTH_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
    REGION_NAME=RegionTwo
    KEYSTONE_REGION_NAME=RegionOne

@@ -685,15 +737,22 @@ KEYSTONE_REGION_NAME to specify the region of Keystone service.
 KEYSTONE_REGION_NAME has a default value the same as REGION_NAME thus we omit
 it in the configuration of RegionOne.

-Disabling Identity API v2
-+++++++++++++++++++++++++
+Glance
++++++

-The Identity API v2 is deprecated as of Mitaka and it is recommended to only
-use the v3 API. It is possible to setup keystone without v2 API, by doing:
+The default image size quota of 1GiB may be too small if larger images
+are to be used. Change the default at setup time with:

 ::

-    ENABLE_IDENTITY_V2=False
+    GLANCE_LIMIT_IMAGE_SIZE_TOTAL=5000
+
+or at runtime via:
+
+::
+
+    openstack --os-cloud devstack-system-admin registered limit set \
+      --service glance --default-limit 5000 --region RegionOne image_size_total

 .. _arch-configuration:

@@ -13,7 +13,7 @@ with Devstack.

 Communication
 ~~~~~~~~~~~~~
-* IRC channel ``#openstack-qa`` at FreeNode
+* IRC channel ``#openstack-qa`` at OFTC.
 * Mailing list (prefix subjects with ``[qa][devstack]`` for faster responses)
  http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-discuss

@@ -42,8 +42,9 @@ Getting Your Patch Merged
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 All changes proposed to the Devstack require two ``Code-Review +2`` votes from
 Devstack core reviewers before one of the core reviewers can approve the patch
-by giving ``Workflow +1`` vote. One exception is for patches to unblock the gate
-which can be approved by single core reviewers.
+by giving ``Workflow +1`` vote. There are 2 exceptions, approving patches to
+unblock the gate and patches that do not relate to the Devstack's core logic,
+like for example old job cleanups, can be approved by single core reviewers.

 Project Team Lead Duties
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -20,6 +20,12 @@ provides consumption output when available memory is seen to be
 falling (i.e. processes are consuming memory).  It also provides
 output showing locked (unswappable) memory.

+file_tracker
+------------
+
+The ``file_tracker`` service periodically monitors the number of
+open files in the system.
+
 tcpdump
 -------

@@ -20,7 +20,7 @@ Walk through various setups used by stackers
   guides/neutron
   guides/devstack-with-nested-kvm
   guides/nova
-   guides/devstack-with-lbaas-v2
+   guides/devstack-with-octavia
   guides/devstack-with-ldap

 All-In-One Single VM
@@ -69,10 +69,10 @@ Nova and devstack

 Guide to working with nova features :doc:`Nova and devstack <guides/nova>`.

-Configure Load-Balancer Version 2
-----------------------------------
+Configure Octavia
+-----------------

-Guide on :doc:`Configure Load-Balancer Version 2 <guides/devstack-with-lbaas-v2>`.
+Guide on :doc:`Configure Octavia <guides/devstack-with-octavia>`.

 Deploying DevStack with LDAP
 ----------------------------
@@ -1,145 +0,0 @@
-Devstack with Octavia Load Balancing
-====================================
-
-Starting with the OpenStack Pike release, Octavia is now a standalone service
-providing load balancing services for OpenStack.
-
-This guide will show you how to create a devstack with `Octavia API`_ enabled.
-
-.. _Octavia API: https://docs.openstack.org/api-ref/load-balancer/v2/index.html
-
-Phase 1: Create DevStack + 2 nova instances
--------------------------------------------
-
-First, set up a vm of your choice with at least 8 GB RAM and 16 GB disk space,
-make sure it is updated. Install git and any other developer tools you find
-useful.
-
-Install devstack
-
-::
-
-    git clone https://opendev.org/openstack/devstack
-    cd devstack/tools
-    sudo ./create-stack-user.sh
-    cd ../..
-    sudo mv devstack /opt/stack
-    sudo chown -R stack.stack /opt/stack/devstack
-
-This will clone the current devstack code locally, then setup the "stack"
-account that devstack services will run under. Finally, it will move devstack
-into its default location in /opt/stack/devstack.
-
-Edit your ``/opt/stack/devstack/local.conf`` to look like
-
-::
-
-    [[local|localrc]]
-    enable_plugin octavia https://opendev.org/openstack/octavia
-    # If you are enabling horizon, include the octavia dashboard
-    # enable_plugin octavia-dashboard https://opendev.org/openstack/octavia-dashboard.git
-    # If you are enabling barbican for TLS offload in Octavia, include it here.
-    # enable_plugin barbican https://opendev.org/openstack/barbican
-
-    # ===== BEGIN localrc =====
-    DATABASE_PASSWORD=password
-    ADMIN_PASSWORD=password
-    SERVICE_PASSWORD=password
-    SERVICE_TOKEN=password
-    RABBIT_PASSWORD=password
-    # Enable Logging
-    LOGFILE=$DEST/logs/stack.sh.log
-    VERBOSE=True
-    LOG_COLOR=True
-    # Pre-requisite
-    ENABLED_SERVICES=rabbit,mysql,key
-    # Horizon - enable for the OpenStack web GUI
-    # ENABLED_SERVICES+=,horizon
-    # Nova
-    ENABLED_SERVICES+=,n-api,n-crt,n-cpu,n-cond,n-sch,n-api-meta,n-sproxy
-    ENABLED_SERVICES+=,placement-api,placement-client
-    # Glance
-    ENABLED_SERVICES+=,g-api
-    # Neutron
-    ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron
-    ENABLED_SERVICES+=,octavia,o-cw,o-hk,o-hm,o-api
-    # Cinder
-    ENABLED_SERVICES+=,c-api,c-vol,c-sch
-    # Tempest
-    ENABLED_SERVICES+=,tempest
-    # Barbican - Optionally used for TLS offload in Octavia
-    # ENABLED_SERVICES+=,barbican
-    # ===== END localrc =====
-
-Run stack.sh and do some sanity checks
-
-::
-
-    sudo su - stack
-    cd /opt/stack/devstack
-    ./stack.sh
-    . ./openrc
-
-    openstack network list  # should show public and private networks
-
-Create two nova instances that we can use as test http servers:
-
-::
-
-    #create nova instances on private network
-    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node1
-    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node2
-    openstack server list # should show the nova instances just created
-
-    #add secgroup rules to allow ssh etc..
-    openstack security group rule create default --protocol icmp
-    openstack security group rule create default --protocol tcp --dst-port 22:22
-    openstack security group rule create default --protocol tcp --dst-port 80:80
-
-Set up a simple web server on each of these instances. ssh into each instance (username 'cirros', password 'cubswin:)' or 'gocubsgo') and run
-
-::
-
-    MYIP=$(ifconfig eth0|grep 'inet addr'|awk -F: '{print $2}'| awk '{print $1}')
-    while true; do echo -e "HTTP/1.0 200 OK\r\n\r\nWelcome to $MYIP" | sudo nc -l -p 80 ; done&
-
-Phase 2: Create your load balancer
----------------------------------
-
-Make sure you have the 'openstack loadbalancer' commands:
-
-::
-
-    pip install python-octaviaclient
-
-Create your load balancer:
-
-::
-
-    openstack loadbalancer create --name lb1 --vip-subnet-id private-subnet
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer listener create --protocol HTTP --protocol-port 80 --name listener1 lb1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer pool create --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer healthmonitor create --delay 5 --timeout 2 --max-retries 1 --type HTTP pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer member create --subnet-id private-subnet --address <web server 1 address> --protocol-port 80 pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer member create --subnet-id private-subnet --address <web server 2 address> --protocol-port 80 pool1
-
-Please note: The <web server # address> fields are the IP addresses of the nova
-servers created in Phase 1.
-Also note, using the API directly you can do all of the above commands in one
-API call.
-
-Phase 3: Test your load balancer
--------------------------------
-
-::
-
-    openstack loadbalancer show lb1 # Note the vip_address
-    curl http://<vip_address>
-    curl http://<vip_address>
-
-This should show the "Welcome to <IP>" message from each member server.
@@ -1,3 +1,5 @@
+.. _kvm_nested_virt:
+
 =======================================================
 Configure DevStack with KVM-based Nested Virtualization
 =======================================================
@@ -0,0 +1,144 @@
+Devstack with Octavia Load Balancing
+====================================
+
+Starting with the OpenStack Pike release, Octavia is now a standalone service
+providing load balancing services for OpenStack.
+
+This guide will show you how to create a devstack with `Octavia API`_ enabled.
+
+.. _Octavia API: https://docs.openstack.org/api-ref/load-balancer/v2/index.html
+
+Phase 1: Create DevStack + 2 nova instances
+--------------------------------------------
+
+First, set up a VM of your choice with at least 8 GB RAM and 16 GB disk space,
+make sure it is updated. Install git and any other developer tools you find
+useful.
+
+Install devstack::
+
+    git clone https://opendev.org/openstack/devstack
+    cd devstack/tools
+    sudo ./create-stack-user.sh
+    cd ../..
+    sudo mv devstack /opt/stack
+    sudo chown -R stack.stack /opt/stack/devstack
+
+This will clone the current devstack code locally, then setup the "stack"
+account that devstack services will run under. Finally, it will move devstack
+into its default location in /opt/stack/devstack.
+
+Edit your ``/opt/stack/devstack/local.conf`` to look like::
+
+    [[local|localrc]]
+    # ===== BEGIN localrc =====
+    DATABASE_PASSWORD=password
+    ADMIN_PASSWORD=password
+    SERVICE_PASSWORD=password
+    SERVICE_TOKEN=password
+    RABBIT_PASSWORD=password
+    GIT_BASE=https://opendev.org
+    # Optional settings:
+    # OCTAVIA_AMP_BASE_OS=centos
+    # OCTAVIA_AMP_DISTRIBUTION_RELEASE_ID=9-stream
+    # OCTAVIA_AMP_IMAGE_SIZE=3
+    # OCTAVIA_LB_TOPOLOGY=ACTIVE_STANDBY
+    # OCTAVIA_ENABLE_AMPHORAV2_JOBBOARD=True
+    # LIBS_FROM_GIT+=octavia-lib,
+    # Enable Logging
+    LOGFILE=$DEST/logs/stack.sh.log
+    VERBOSE=True
+    LOG_COLOR=True
+    enable_service rabbit
+    enable_plugin neutron $GIT_BASE/openstack/neutron
+    # Octavia supports using QoS policies on the VIP port:
+    enable_service q-qos
+    enable_service placement-api placement-client
+    # Octavia services
+    enable_plugin octavia $GIT_BASE/openstack/octavia master
+    enable_plugin octavia-dashboard $GIT_BASE/openstack/octavia-dashboard
+    enable_plugin ovn-octavia-provider $GIT_BASE/openstack/ovn-octavia-provider
+    enable_plugin octavia-tempest-plugin $GIT_BASE/openstack/octavia-tempest-plugin
+    enable_service octavia o-api o-cw o-hm o-hk o-da
+    # If you are enabling barbican for TLS offload in Octavia, include it here.
+    # enable_plugin barbican $GIT_BASE/openstack/barbican
+    # enable_service barbican
+    # Cinder (optional)
+    disable_service c-api c-vol c-sch
+    # Tempest
+    enable_service tempest
+    # ===== END localrc =====
+
+.. note::
+    For best performance it is highly recommended to use KVM
+    virtualization instead of QEMU.
+    Also make sure nested virtualization is enabled as documented in
+    :ref:`the respective guide <kvm_nested_virt>`.
+    By adding ``LIBVIRT_CPU_MODE="host-passthrough"`` to your
+    ``local.conf`` you enable the guest VMs to make use of all features your
+    host's CPU provides.
+
+Run stack.sh and do some sanity checks::
+
+    sudo su - stack
+    cd /opt/stack/devstack
+    ./stack.sh
+    . ./openrc
+
+    openstack network list  # should show public and private networks
+
+Create two nova instances that we can use as test http servers::
+
+    # create nova instances on private network
+    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node1
+    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node2
+    openstack server list # should show the nova instances just created
+
+    # add secgroup rules to allow ssh etc..
+    openstack security group rule create default --protocol icmp
+    openstack security group rule create default --protocol tcp --dst-port 22:22
+    openstack security group rule create default --protocol tcp --dst-port 80:80
+
+Set up a simple web server on each of these instances. One possibility is to use
+the `Golang test server`_ that is used by the Octavia project for CI testing
+as well.
+Copy the binary to your instances and start it as shown below
+(username 'cirros', password 'gocubsgo')::
+
+    INST_IP=<instance IP>
+    scp -O test_server.bin cirros@${INST_IP}:
+    ssh -f cirros@${INST_IP} ./test_server.bin -id ${INST_IP}
+
+When started this way the test server will respond to HTTP requests with
+its own IP.
+
+Phase 2: Create your load balancer
+----------------------------------
+
+Create your load balancer::
+
+    openstack loadbalancer create --wait --name lb1 --vip-subnet-id private-subnet
+    openstack loadbalancer listener create --wait --protocol HTTP --protocol-port 80 --name listener1 lb1
+    openstack loadbalancer pool create --wait --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
+    openstack loadbalancer healthmonitor create --wait --delay 5 --timeout 2 --max-retries 1 --type HTTP pool1
+    openstack loadbalancer member create --wait --subnet-id private-subnet --address <web server 1 address> --protocol-port 80 pool1
+    openstack loadbalancer member create --wait --subnet-id private-subnet --address <web server 2 address> --protocol-port 80 pool1
+
+Please note: The <web server # address> fields are the IP addresses of the nova
+servers created in Phase 1.
+Also note, using the API directly you can do all of the above commands in one
+API call.
+
+Phase 3: Test your load balancer
+--------------------------------
+
+::
+
+    openstack loadbalancer show lb1 # Note the vip_address
+    curl http://<vip_address>
+    curl http://<vip_address>
+
+This should show the "Welcome to <IP>" message from each member server.
+
+
+.. _Golang test server: https://opendev.org/openstack/octavia-tempest-plugin/src/branch/master/octavia_tempest_plugin/contrib/test_server
@@ -75,13 +75,21 @@ Otherwise create the stack user:

    useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+::
+
+    chmod +x /opt/stack
+
 This user will be making many changes to your system during installation
 and operation so it needs to have sudo privileges to root without a
 password:

 ::

-    echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 From here on use the ``stack`` user. **Logout** and **login** as the
 ``stack`` user.
@@ -169,7 +177,7 @@ machines, create a ``local.conf`` with:
    MYSQL_HOST=$SERVICE_HOST
    RABBIT_HOST=$SERVICE_HOST
    GLANCE_HOSTPORT=$SERVICE_HOST:9292
-    ENABLED_SERVICES=n-cpu,q-agt,c-vol,placement-client
+    ENABLED_SERVICES=n-cpu,c-vol,placement-client,ovn-controller,ovs-vswitchd,ovsdb-server,q-ovn-metadata-agent
    NOVA_VNC_ENABLED=True
    NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_lite.html"
    VNCSERVER_LISTEN=$HOST_IP
@@ -395,7 +403,7 @@ SSH keys need to be exchanged between each compute node:

 3. Verify that login via ssh works without a password::

-        ssh -i /root/.ssh/id_rsa.pub stack@DESTINATION
+        ssh -i /root/.ssh/id_rsa stack@DESTINATION

 In essence, this means that every compute node's root user's public RSA key
 must exist in every other compute node's stack user's authorized_keys file and
@@ -41,19 +41,8 @@ network and is on a shared subnet with other machines.  The
 `local.conf` exhibited here assumes that 1500 is a reasonable MTU to
 use on that network.

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                }
-        }
+.. image:: /assets/images/neutron-network-1.png
+   :alt: Network configuration for a single DevStack node


 DevStack Configuration
@@ -100,21 +89,8 @@ also want to do multinode testing and networking.
 Physical Network Setup
 ~~~~~~~~~~~~~~~~~~~~~~

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                        devstack-2 [ address = "172.18.161.7" ];
-                }
-        }
-
+.. image:: /assets/images/neutron-network-2.png
+   :alt: Network configuration for multiple DevStack nodes

 After DevStack installs and configures Neutron, traffic from guest VMs
 flows out of `devstack-2` (the compute node) and is encapsulated in a
@@ -222,8 +198,6 @@ connect OpenStack nodes (like `devstack-2`) together. This bridge is
 used so that project network traffic, using the VXLAN tunneling
 protocol, flows between each compute node where project instances run.

-
-
 DevStack Compute Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -268,30 +242,8 @@ to the neutron L3 service.
 Physical Network Setup
 ----------------------

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network provider_net {
-                        address = "203.0.113.0/24"
-                        router [ address = "203.0.113.1" ];
-                        controller;
-                        compute1;
-                        compute2;
-                }
-
-                network control_plane {
-                        router [ address = "10.0.0.1" ]
-                        address = "10.0.0.0/24"
-                        controller [ address = "10.0.0.2" ]
-                        compute1 [ address = "10.0.0.3" ]
-                        compute2 [ address = "10.0.0.4" ]
-                }
-        }
-
+.. image:: /assets/images/neutron-network-3.png
+   :alt: Network configuration for provider networks

 On a compute node, the first interface, eth0 is used for the OpenStack
 management (API, message bus, etc) as well as for ssh for an
@@ -122,7 +122,7 @@ when creating the server, for example:
 .. code-block:: shell

  $ openstack --os-compute-api-version 2.37 server create --flavor cirros256 \
-      --image cirros-0.3.5-x86_64-disk --nic none --wait test-server
+      --image cirros-0.6.3-x86_64-disk --nic none --wait test-server

 .. note:: ``--os-compute-api-version`` greater than or equal to 2.37 is
          required to use ``--nic=none``.
@@ -49,13 +49,21 @@ below)

    $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it will need
 to have sudo privileges:

 .. code-block:: console

    $ apt-get install sudo -y || yum install -y sudo
-    $ echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    $ echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 .. note:: On some systems you may need to use ``sudo visudo``.

@@ -98,6 +106,9 @@ do the following:
 -  Set the service password. This is used by the OpenStack services
   (Nova, Glance, etc) to authenticate with Keystone.

+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.
+
 ``local.conf`` should look something like this:

 .. code-block:: ini
@@ -37,10 +37,10 @@ Install Linux
 -------------

 Start with a clean and minimal install of a Linux system. DevStack
-attempts to support the two latest LTS releases of Ubuntu, the
-latest/current Fedora version, CentOS/RHEL 8 and OpenSUSE.
+attempts to support the two latest LTS releases of Ubuntu,
+Rocky Linux 9 and openEuler.

-If you do not have a preference, Ubuntu 18.04 (Bionic Beaver) is the
+If you do not have a preference, Ubuntu 22.04 (Jammy) is the
 most tested, and will probably go the smoothest.

 Add Stack User (optional)
@@ -57,13 +57,21 @@ to run DevStack with

   $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it should
 have sudo privileges:

 .. code-block:: console

    $ echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack
-    $ sudo su - stack
+    $ sudo -u stack -i

 Download DevStack
 -----------------
@@ -93,7 +101,10 @@ devstack git repo.
 This is the minimum required config to get started with DevStack.

 .. note:: There is a sample :download:`local.conf </assets/local.conf>` file
-    under the *samples* directory in the devstack repository.
+   under the *samples* directory in the devstack repository.
+
+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.

 Start the install
 -----------------
@@ -102,7 +113,7 @@ Start the install

   $ ./stack.sh

-This will take a 15 - 20 minutes, largely depending on the speed of
+This will take 15 - 30 minutes, largely depending on the speed of
 your internet connection. Many git trees and packages will be
 installed during this process.

@@ -122,6 +133,8 @@ there.
 You can ``source openrc`` in your shell, and then use the
 ``openstack`` command line tool to manage your devstack.

+You can :ref:`create a VM and SSH into it <ssh>`.
+
 You can ``cd /opt/stack/tempest`` and run tempest tests that have
 been configured to work with your devstack.

@@ -68,7 +68,7 @@ Shared Guest Interface
 .. warning::

   This is not a recommended configuration. Because of interactions
-   between ovs and bridging, if you reboot your box with active
+   between OVS and bridging, if you reboot your box with active
   networking you may lose network connectivity to your system.

 If you need your guests accessible on the network, but only have 1
@@ -114,3 +114,125 @@ For IPv6, ``FIXED_RANGE_V6`` will default to the first /64 of the value of
 ``FIXED_RANGE_V6`` will just use the value of that directly.
 ``SUBNETPOOL_PREFIX_V6`` will just default to the value of
 ``IPV6_ADDRS_SAFE_TO_USE`` directly.
+
+.. _ssh:
+
+SSH access to instances
+=======================
+
+To validate connectivity, you can create an instance using the
+``$PRIVATE_NETWORK_NAME`` network (default: ``private``), create a floating IP
+using the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``), and attach
+this floating IP to the instance:
+
+.. code-block:: shell
+
+    openstack keypair create --public-key ~/.ssh/id_rsa.pub test-keypair
+    openstack server create --network private --key-name test-keypair ... test-server
+    fip_id=$(openstack floating ip create public -f value -c id)
+    openstack server add floating ip test-server ${fip_id}
+
+Once done, ensure you have enabled SSH and ICMP (ping) access for the security
+group used for the instance. You can either create a custom security group and
+specify it when creating the instance or add it after creation, or you can
+modify the ``default`` security group created by default for each project.
+Let's do the latter:
+
+.. code-block:: shell
+
+    openstack security group rule create --proto icmp --dst-port 0 default
+    openstack security group rule create --proto tcp --dst-port 22 default
+
+Finally, SSH into the instance. If you used the Cirros instance uploaded by
+default, then you can run the following:
+
+.. code-block:: shell
+
+    openstack server ssh test-server -- -l cirros
+
+This will connect using the ``cirros`` user and the keypair you configured when
+creating the instance.
+
+Remote SSH access to instances
+==============================
+
+You can also SSH to created instances on your DevStack host from other hosts.
+This can be helpful if you are e.g. deploying DevStack in a VM on an existing
+cloud and wish to do development on your local machine. There are a few ways to
+do this.
+
+.. rubric:: Configure instances to be locally accessible
+
+The most obvious way is to configure guests to be locally accessible, as
+described `above <Locally Accessible Guests>`__. This has the advantage of
+requiring no further effort on the client. However, it is more involved and
+requires either support from your cloud or some inadvisable workarounds.
+
+.. rubric:: Use your DevStack host as a jump host
+
+You can choose to use your DevStack host as a jump host. To SSH to a instance
+this way, pass the standard ``-J`` option to the ``openstack ssh`` / ``ssh``
+command. For example:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -J username@devstack-host
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+This can also be configured via your ``~/.ssh/config`` file, making it rather
+effortless. However, it only allows SSH access. If you want to access e.g. a
+web application on the instance, you will need to configure an SSH tunnel and
+forward select ports using the ``-L`` option. For example, to forward HTTP
+traffic:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -L 8080:username@devstack-host:80
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+As you can imagine, this can quickly get out of hand, particularly for more
+complex guest applications with multiple ports.
+
+.. rubric:: Use a proxy or VPN tool
+
+You can use a proxy or VPN tool to enable tunneling for the floating IP
+address range of the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``)
+defined by ``$FLOATING_RANGE`` (default: ``172.24.4.0/24``). There are many
+such tools available to do this. For example, we could use a useful utility
+called `shuttle`__. To enable tunneling using ``shuttle``, first ensure you
+have allowed SSH and HTTP(S) traffic to your DevStack host. Allowing HTTP(S)
+traffic is necessary so you can use the OpenStack APIs remotely. How you do
+this will depend on where your DevStack host is running. Once this is done,
+install ``sshuttle`` on your localhost:
+
+.. code-block:: bash
+
+    sudo apt-get install sshuttle || yum install sshuttle
+
+Finally, start ``sshuttle`` on your localhost using the floating IP address
+range. For example, assuming you are using the default value for
+``$FLOATING_RANGE``, you can do:
+
+.. code-block:: bash
+
+    sshuttle -r username@devstack-host 172.24.4.0/24
+
+(where ``username`` and ``devstack-host`` are the username and hostname of your
+DevStack host).
+
+You should now be able to create an instance and SSH into it:
+
+.. code-block:: bash
+
+    openstack server ssh test-server -- -l cirros
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`)
+
+.. __: https://github.com/sshuttle/sshuttle
@@ -23,13 +23,12 @@ strategy to include the latest Ubuntu release and the latest RHEL
 release.*

 -  Ubuntu: current LTS release plus current development release
-  Fedora: current release plus previous release
-  RHEL/CentOS: current major release
+-  RHEL/CentOS/RockyLinux: current major release
 -  Other OS platforms may continue to be included but the maintenance of
   those platforms shall not be assumed simply due to their presence.
   Having a listed point-of-contact for each additional OS will greatly
   increase its chance of being well-maintained.
-  Patches for Ubuntu and/or Fedora will not be held up due to
+-  Patches for Ubuntu and/or RockyLinux will not be held up due to
   side-effects on other OS platforms.

 Databases
@@ -28,8 +28,6 @@ openstack/aodh                           `https://opendev.org/openstack/aodh <ht
 openstack/barbican                       `https://opendev.org/openstack/barbican <https://opendev.org/openstack/barbican>`__
 openstack/blazar                         `https://opendev.org/openstack/blazar <https://opendev.org/openstack/blazar>`__
 openstack/ceilometer                     `https://opendev.org/openstack/ceilometer <https://opendev.org/openstack/ceilometer>`__
-openstack/ceilometer-powervm             `https://opendev.org/openstack/ceilometer-powervm <https://opendev.org/openstack/ceilometer-powervm>`__
-openstack/cinderlib                      `https://opendev.org/openstack/cinderlib <https://opendev.org/openstack/cinderlib>`__
 openstack/cloudkitty                     `https://opendev.org/openstack/cloudkitty <https://opendev.org/openstack/cloudkitty>`__
 openstack/cyborg                         `https://opendev.org/openstack/cyborg <https://opendev.org/openstack/cyborg>`__
 openstack/designate                      `https://opendev.org/openstack/designate <https://opendev.org/openstack/designate>`__
@@ -39,9 +37,6 @@ openstack/devstack-plugin-container      `https://opendev.org/openstack/devstack
 openstack/devstack-plugin-kafka          `https://opendev.org/openstack/devstack-plugin-kafka <https://opendev.org/openstack/devstack-plugin-kafka>`__
 openstack/devstack-plugin-nfs            `https://opendev.org/openstack/devstack-plugin-nfs <https://opendev.org/openstack/devstack-plugin-nfs>`__
 openstack/devstack-plugin-open-cas       `https://opendev.org/openstack/devstack-plugin-open-cas <https://opendev.org/openstack/devstack-plugin-open-cas>`__
-openstack/devstack-plugin-pika           `https://opendev.org/openstack/devstack-plugin-pika <https://opendev.org/openstack/devstack-plugin-pika>`__
-openstack/devstack-plugin-zmq            `https://opendev.org/openstack/devstack-plugin-zmq <https://opendev.org/openstack/devstack-plugin-zmq>`__
-openstack/ec2-api                        `https://opendev.org/openstack/ec2-api <https://opendev.org/openstack/ec2-api>`__
 openstack/freezer                        `https://opendev.org/openstack/freezer <https://opendev.org/openstack/freezer>`__
 openstack/freezer-api                    `https://opendev.org/openstack/freezer-api <https://opendev.org/openstack/freezer-api>`__
 openstack/freezer-tempest-plugin         `https://opendev.org/openstack/freezer-tempest-plugin <https://opendev.org/openstack/freezer-tempest-plugin>`__
@@ -52,12 +47,8 @@ openstack/ironic                         `https://opendev.org/openstack/ironic <
 openstack/ironic-inspector               `https://opendev.org/openstack/ironic-inspector <https://opendev.org/openstack/ironic-inspector>`__
 openstack/ironic-prometheus-exporter     `https://opendev.org/openstack/ironic-prometheus-exporter <https://opendev.org/openstack/ironic-prometheus-exporter>`__
 openstack/ironic-ui                      `https://opendev.org/openstack/ironic-ui <https://opendev.org/openstack/ironic-ui>`__
-openstack/karbor                         `https://opendev.org/openstack/karbor <https://opendev.org/openstack/karbor>`__
-openstack/karbor-dashboard               `https://opendev.org/openstack/karbor-dashboard <https://opendev.org/openstack/karbor-dashboard>`__
 openstack/keystone                       `https://opendev.org/openstack/keystone <https://opendev.org/openstack/keystone>`__
-openstack/kuryr-kubernetes               `https://opendev.org/openstack/kuryr-kubernetes <https://opendev.org/openstack/kuryr-kubernetes>`__
 openstack/kuryr-libnetwork               `https://opendev.org/openstack/kuryr-libnetwork <https://opendev.org/openstack/kuryr-libnetwork>`__
-openstack/kuryr-tempest-plugin           `https://opendev.org/openstack/kuryr-tempest-plugin <https://opendev.org/openstack/kuryr-tempest-plugin>`__
 openstack/magnum                         `https://opendev.org/openstack/magnum <https://opendev.org/openstack/magnum>`__
 openstack/magnum-ui                      `https://opendev.org/openstack/magnum-ui <https://opendev.org/openstack/magnum-ui>`__
 openstack/manila                         `https://opendev.org/openstack/manila <https://opendev.org/openstack/manila>`__
@@ -65,55 +56,40 @@ openstack/manila-tempest-plugin          `https://opendev.org/openstack/manila-t
 openstack/manila-ui                      `https://opendev.org/openstack/manila-ui <https://opendev.org/openstack/manila-ui>`__
 openstack/masakari                       `https://opendev.org/openstack/masakari <https://opendev.org/openstack/masakari>`__
 openstack/mistral                        `https://opendev.org/openstack/mistral <https://opendev.org/openstack/mistral>`__
-openstack/monasca-analytics              `https://opendev.org/openstack/monasca-analytics <https://opendev.org/openstack/monasca-analytics>`__
 openstack/monasca-api                    `https://opendev.org/openstack/monasca-api <https://opendev.org/openstack/monasca-api>`__
-openstack/monasca-ceilometer             `https://opendev.org/openstack/monasca-ceilometer <https://opendev.org/openstack/monasca-ceilometer>`__
 openstack/monasca-events-api             `https://opendev.org/openstack/monasca-events-api <https://opendev.org/openstack/monasca-events-api>`__
-openstack/monasca-log-api                `https://opendev.org/openstack/monasca-log-api <https://opendev.org/openstack/monasca-log-api>`__
 openstack/monasca-tempest-plugin         `https://opendev.org/openstack/monasca-tempest-plugin <https://opendev.org/openstack/monasca-tempest-plugin>`__
-openstack/monasca-transform              `https://opendev.org/openstack/monasca-transform <https://opendev.org/openstack/monasca-transform>`__
-openstack/murano                         `https://opendev.org/openstack/murano <https://opendev.org/openstack/murano>`__
 openstack/networking-bagpipe             `https://opendev.org/openstack/networking-bagpipe <https://opendev.org/openstack/networking-bagpipe>`__
 openstack/networking-baremetal           `https://opendev.org/openstack/networking-baremetal <https://opendev.org/openstack/networking-baremetal>`__
 openstack/networking-bgpvpn              `https://opendev.org/openstack/networking-bgpvpn <https://opendev.org/openstack/networking-bgpvpn>`__
 openstack/networking-generic-switch      `https://opendev.org/openstack/networking-generic-switch <https://opendev.org/openstack/networking-generic-switch>`__
-openstack/networking-hyperv              `https://opendev.org/openstack/networking-hyperv <https://opendev.org/openstack/networking-hyperv>`__
-openstack/networking-l2gw                `https://opendev.org/openstack/networking-l2gw <https://opendev.org/openstack/networking-l2gw>`__
-openstack/networking-midonet             `https://opendev.org/openstack/networking-midonet <https://opendev.org/openstack/networking-midonet>`__
-openstack/networking-odl                 `https://opendev.org/openstack/networking-odl <https://opendev.org/openstack/networking-odl>`__
-openstack/networking-powervm             `https://opendev.org/openstack/networking-powervm <https://opendev.org/openstack/networking-powervm>`__
 openstack/networking-sfc                 `https://opendev.org/openstack/networking-sfc <https://opendev.org/openstack/networking-sfc>`__
 openstack/neutron                        `https://opendev.org/openstack/neutron <https://opendev.org/openstack/neutron>`__
 openstack/neutron-dynamic-routing        `https://opendev.org/openstack/neutron-dynamic-routing <https://opendev.org/openstack/neutron-dynamic-routing>`__
+openstack/neutron-fwaas                  `https://opendev.org/openstack/neutron-fwaas <https://opendev.org/openstack/neutron-fwaas>`__
+openstack/neutron-fwaas-dashboard        `https://opendev.org/openstack/neutron-fwaas-dashboard <https://opendev.org/openstack/neutron-fwaas-dashboard>`__
 openstack/neutron-tempest-plugin         `https://opendev.org/openstack/neutron-tempest-plugin <https://opendev.org/openstack/neutron-tempest-plugin>`__
 openstack/neutron-vpnaas                 `https://opendev.org/openstack/neutron-vpnaas <https://opendev.org/openstack/neutron-vpnaas>`__
 openstack/neutron-vpnaas-dashboard       `https://opendev.org/openstack/neutron-vpnaas-dashboard <https://opendev.org/openstack/neutron-vpnaas-dashboard>`__
-openstack/nova-powervm                   `https://opendev.org/openstack/nova-powervm <https://opendev.org/openstack/nova-powervm>`__
+openstack/nova                           `https://opendev.org/openstack/nova <https://opendev.org/openstack/nova>`__
 openstack/octavia                        `https://opendev.org/openstack/octavia <https://opendev.org/openstack/octavia>`__
 openstack/octavia-dashboard              `https://opendev.org/openstack/octavia-dashboard <https://opendev.org/openstack/octavia-dashboard>`__
 openstack/octavia-tempest-plugin         `https://opendev.org/openstack/octavia-tempest-plugin <https://opendev.org/openstack/octavia-tempest-plugin>`__
 openstack/openstacksdk                   `https://opendev.org/openstack/openstacksdk <https://opendev.org/openstack/openstacksdk>`__
-openstack/os-loganalyze                  `https://opendev.org/openstack/os-loganalyze <https://opendev.org/openstack/os-loganalyze>`__
 openstack/osprofiler                     `https://opendev.org/openstack/osprofiler <https://opendev.org/openstack/osprofiler>`__
-openstack/oswin-tempest-plugin           `https://opendev.org/openstack/oswin-tempest-plugin <https://opendev.org/openstack/oswin-tempest-plugin>`__
+openstack/ovn-bgp-agent                  `https://opendev.org/openstack/ovn-bgp-agent <https://opendev.org/openstack/ovn-bgp-agent>`__
 openstack/ovn-octavia-provider           `https://opendev.org/openstack/ovn-octavia-provider <https://opendev.org/openstack/ovn-octavia-provider>`__
-openstack/panko                          `https://opendev.org/openstack/panko <https://opendev.org/openstack/panko>`__
-openstack/patrole                        `https://opendev.org/openstack/patrole <https://opendev.org/openstack/patrole>`__
-openstack/qinling                        `https://opendev.org/openstack/qinling <https://opendev.org/openstack/qinling>`__
-openstack/qinling-dashboard              `https://opendev.org/openstack/qinling-dashboard <https://opendev.org/openstack/qinling-dashboard>`__
 openstack/rally-openstack                `https://opendev.org/openstack/rally-openstack <https://opendev.org/openstack/rally-openstack>`__
-openstack/sahara                         `https://opendev.org/openstack/sahara <https://opendev.org/openstack/sahara>`__
-openstack/sahara-dashboard               `https://opendev.org/openstack/sahara-dashboard <https://opendev.org/openstack/sahara-dashboard>`__
-openstack/searchlight                    `https://opendev.org/openstack/searchlight <https://opendev.org/openstack/searchlight>`__
-openstack/searchlight-ui                 `https://opendev.org/openstack/searchlight-ui <https://opendev.org/openstack/searchlight-ui>`__
-openstack/senlin                         `https://opendev.org/openstack/senlin <https://opendev.org/openstack/senlin>`__
 openstack/shade                          `https://opendev.org/openstack/shade <https://opendev.org/openstack/shade>`__
-openstack/solum                          `https://opendev.org/openstack/solum <https://opendev.org/openstack/solum>`__
+openstack/skyline-apiserver              `https://opendev.org/openstack/skyline-apiserver <https://opendev.org/openstack/skyline-apiserver>`__
 openstack/storlets                       `https://opendev.org/openstack/storlets <https://opendev.org/openstack/storlets>`__
 openstack/tacker                         `https://opendev.org/openstack/tacker <https://opendev.org/openstack/tacker>`__
+openstack/tap-as-a-service               `https://opendev.org/openstack/tap-as-a-service <https://opendev.org/openstack/tap-as-a-service>`__
 openstack/telemetry-tempest-plugin       `https://opendev.org/openstack/telemetry-tempest-plugin <https://opendev.org/openstack/telemetry-tempest-plugin>`__
 openstack/trove                          `https://opendev.org/openstack/trove <https://opendev.org/openstack/trove>`__
 openstack/trove-dashboard                `https://opendev.org/openstack/trove-dashboard <https://opendev.org/openstack/trove-dashboard>`__
+openstack/venus                          `https://opendev.org/openstack/venus <https://opendev.org/openstack/venus>`__
+openstack/venus-dashboard                `https://opendev.org/openstack/venus-dashboard <https://opendev.org/openstack/venus-dashboard>`__
 openstack/vitrage                        `https://opendev.org/openstack/vitrage <https://opendev.org/openstack/vitrage>`__
 openstack/vitrage-dashboard              `https://opendev.org/openstack/vitrage-dashboard <https://opendev.org/openstack/vitrage-dashboard>`__
 openstack/vitrage-tempest-plugin         `https://opendev.org/openstack/vitrage-tempest-plugin <https://opendev.org/openstack/vitrage-tempest-plugin>`__
@@ -143,6 +119,7 @@ x/devstack-plugin-glusterfs              `https://opendev.org/x/devstack-plugin-
 x/devstack-plugin-hdfs                   `https://opendev.org/x/devstack-plugin-hdfs <https://opendev.org/x/devstack-plugin-hdfs>`__
 x/devstack-plugin-libvirt-qemu           `https://opendev.org/x/devstack-plugin-libvirt-qemu <https://opendev.org/x/devstack-plugin-libvirt-qemu>`__
 x/devstack-plugin-mariadb                `https://opendev.org/x/devstack-plugin-mariadb <https://opendev.org/x/devstack-plugin-mariadb>`__
+x/devstack-plugin-tobiko                 `https://opendev.org/x/devstack-plugin-tobiko <https://opendev.org/x/devstack-plugin-tobiko>`__
 x/devstack-plugin-vmax                   `https://opendev.org/x/devstack-plugin-vmax <https://opendev.org/x/devstack-plugin-vmax>`__
 x/drbd-devstack                          `https://opendev.org/x/drbd-devstack <https://opendev.org/x/drbd-devstack>`__
 x/fenix                                  `https://opendev.org/x/fenix <https://opendev.org/x/fenix>`__
@@ -169,6 +146,7 @@ x/networking-fortinet                    `https://opendev.org/x/networking-forti
 x/networking-hpe                         `https://opendev.org/x/networking-hpe <https://opendev.org/x/networking-hpe>`__
 x/networking-huawei                      `https://opendev.org/x/networking-huawei <https://opendev.org/x/networking-huawei>`__
 x/networking-infoblox                    `https://opendev.org/x/networking-infoblox <https://opendev.org/x/networking-infoblox>`__
+x/networking-l2gw                        `https://opendev.org/x/networking-l2gw <https://opendev.org/x/networking-l2gw>`__
 x/networking-lagopus                     `https://opendev.org/x/networking-lagopus <https://opendev.org/x/networking-lagopus>`__
 x/networking-mlnx                        `https://opendev.org/x/networking-mlnx <https://opendev.org/x/networking-mlnx>`__
 x/networking-nec                         `https://opendev.org/x/networking-nec <https://opendev.org/x/networking-nec>`__
@@ -190,14 +168,13 @@ x/rsd-virt-for-nova                      `https://opendev.org/x/rsd-virt-for-nov
 x/scalpels                               `https://opendev.org/x/scalpels <https://opendev.org/x/scalpels>`__
 x/slogging                               `https://opendev.org/x/slogging <https://opendev.org/x/slogging>`__
 x/stackube                               `https://opendev.org/x/stackube <https://opendev.org/x/stackube>`__
-x/tap-as-a-service                       `https://opendev.org/x/tap-as-a-service <https://opendev.org/x/tap-as-a-service>`__
 x/tap-as-a-service-dashboard             `https://opendev.org/x/tap-as-a-service-dashboard <https://opendev.org/x/tap-as-a-service-dashboard>`__
 x/tatu                                   `https://opendev.org/x/tatu <https://opendev.org/x/tatu>`__
-x/tobiko                                 `https://opendev.org/x/tobiko <https://opendev.org/x/tobiko>`__
 x/trio2o                                 `https://opendev.org/x/trio2o <https://opendev.org/x/trio2o>`__
 x/valet                                  `https://opendev.org/x/valet <https://opendev.org/x/valet>`__
 x/vmware-nsx                             `https://opendev.org/x/vmware-nsx <https://opendev.org/x/vmware-nsx>`__
 x/vmware-vspc                            `https://opendev.org/x/vmware-vspc <https://opendev.org/x/vmware-vspc>`__
+x/whitebox-neutron-tempest-plugin        `https://opendev.org/x/whitebox-neutron-tempest-plugin <https://opendev.org/x/whitebox-neutron-tempest-plugin>`__
 ======================================== ===


@@ -238,13 +238,10 @@ package dependencies, packages may be listed at the following
 locations in the top-level of the plugin repository:

 - ``./devstack/files/debs/$plugin_name`` - Packages to install when running
-  on Ubuntu, Debian or Linux Mint.
+  on Ubuntu or Debian.

 - ``./devstack/files/rpms/$plugin_name`` - Packages to install when running
-  on Red Hat, Fedora, CentOS or XenServer.
-
- ``./devstack/files/rpms-suse/$plugin_name`` - Packages to install when
-  running on SUSE Linux or openSUSE.
+  on Red Hat, Fedora, or CentOS.

 Although there a no plans to remove this method of installing
 packages, plugins should consider it deprecated for ``bindep`` support
@@ -196,31 +196,6 @@ See the `remote-pdb`_ home page for more options.

 .. _`remote-pdb`: https://pypi.org/project/remote-pdb/

-Known Issues
-============
-
-Be careful about systemd python libraries. There are 3 of them on
-pypi, and they are all very different. They unfortunately all install
-into the ``systemd`` namespace, which can cause some issues.
-
- ``systemd-python`` - this is the upstream maintained library, it has
-  a version number like systemd itself (currently ``234``). This is
-  the one you want.
- ``systemd`` - a python 3 only library, not what you want.
- ``python-systemd`` - another library you don't want. Installing it
-  on a system will break ansible's ability to run. The package has now
-  been renamed to ``cysystemd``, which avoids the namespace collision.
-
-
-If we were using user units, the ``[Service]`` - ``Group=`` parameter
-doesn't seem to work with user units, even though the documentation
-says that it should. This means that we will need to do an explicit
-``/usr/bin/sg``. This has the downside of making the SYSLOG_IDENTIFIER
-be ``sg``. We can explicitly set that with ``SyslogIdentifier=``, but
-it's really unfortunate that we're going to need this work
-around. This is currently not a problem because we're only using
-system units.
-
 Future Work
 ===========

@@ -0,0 +1,25 @@
+=======
+Tempest
+=======
+
+`Tempest`_ is the OpenStack Integration test suite. It is installed by default
+and is used to provide integration testing for many of the OpenStack services.
+Just like DevStack itself, it is possible to extend Tempest with plugins. In
+fact, many Tempest plugin packages also include DevStack plugin to do things
+like pre-create required static resources.
+
+The `Tempest documentation <Tempest>`_ provides a thorough guide to using
+Tempest. However, if you simply wish to run the standard set of Tempest tests
+against an existing deployment, you can do the following:
+
+.. code-block:: shell
+
+    cd /opt/stack/tempest
+    /opt/stack/data/venv/bin/tempest run ...
+
+The above assumes you have installed DevStack in the default location
+(configured via the ``DEST`` configuration variable) and have enabled
+virtualenv-based installation in the standard location (configured via the
+``USE_VENV`` and ``VENV_DEST`` configuration variables, respectively).
+
+.. _Tempest: https://docs.openstack.org/tempest/latest/
@@ -6,7 +6,7 @@ if is_service_enabled tempest; then
        source $TOP_DIR/lib/tempest
    elif [[ "$1" == "stack" && "$2" == "install" ]]; then
        echo_summary "Installing Tempest"
-        install_tempest
+        async_runfunc install_tempest
    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
        # Tempest config must come after layer 2 services are running
        :
@@ -17,6 +17,7 @@ if is_service_enabled tempest; then
        # local.conf Tempest option overrides
        :
    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
+        async_wait install_tempest
        echo_summary "Initializing Tempest"
        configure_tempest
        echo_summary "Installing Tempest Plugins"
@@ -39,4 +39,5 @@
    CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined
 </VirtualHost>

+%WSGIPYTHONHOME%
 WSGISocketPrefix /var/run/%APACHE_NAME%
@@ -1,5 +1,4 @@
 Listen %PUBLICPORT%
-Listen %ADMINPORT%
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" keystone_combined

 <Directory %KEYSTONE_BIN%>
@@ -20,24 +19,11 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
    %SSLKEYFILE%
 </VirtualHost>

-<VirtualHost *:%ADMINPORT%>
-    WSGIDaemonProcess keystone-admin processes=3 threads=1 user=%USER% display-name=%{GROUP} %VIRTUALENV%
-    WSGIProcessGroup keystone-admin
-    WSGIScriptAlias / %KEYSTONE_BIN%/keystone-wsgi-admin
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-    ErrorLogFormat "%M"
-    ErrorLog /var/log/%APACHE_NAME%/keystone.log
-    CustomLog /var/log/%APACHE_NAME%/keystone_access.log keystone_combined
-    %SSLENGINE%
-    %SSLCERTFILE%
-    %SSLKEYFILE%
-</VirtualHost>
-
 %SSLLISTEN%<VirtualHost *:443>
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
 %SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%    SSLProtocol -all +TLSv1.3 +TLSv1.2
 %SSLLISTEN%</VirtualHost>

 Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
@@ -49,13 +35,3 @@ Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
    WSGIApplicationGroup %{GLOBAL}
    WSGIPassAuthorization On
 </Location>
-
-Alias /identity_admin %KEYSTONE_BIN%/keystone-wsgi-admin
-<Location /identity_admin>
-    SetHandler wsgi-script
-    Options +ExecCGI
-
-    WSGIProcessGroup keystone-admin
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-</Location>
@@ -24,6 +24,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
 %SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%    SSLProtocol -all +TLSv1.3 +TLSv1.2
 %SSLLISTEN%</VirtualHost>

 Alias /networking %NEUTRON_BIN%/neutron-api
@@ -1 +0,0 @@
-debs/
@@ -1 +1,2 @@
-dstat
+dstat # dist:bionic
+pcp
@@ -5,6 +5,7 @@ bsdmainutils
 curl
 default-jre-headless  # NOPRIME
 g++
+gawk
 gcc
 gettext  # used for compiling message catalogs
 git
@@ -13,7 +14,6 @@ iputils-ping
 libapache2-mod-proxy-uwsgi
 libffi-dev # for pyOpenSSL
 libjpeg-dev # Pillow 3.0.0
-libmysqlclient-dev  # MySQL-python
 libpcre3-dev # for python-pcre
 libpq-dev  # psycopg2
 libssl-dev # for pyOpenSSL
@@ -28,6 +28,7 @@ pkg-config
 psmisc
 python3-dev
 python3-pip
+python3-systemd
 python3-venv
 tar
 tcpdump
@@ -6,7 +6,6 @@ haproxy # to serve as metadata proxy inside router/dhcp namespaces
 iptables
 iputils-arping
 iputils-ping
-libmysqlclient-dev
 mysql-server #NOPRIME
 postgresql-server-dev-all
 python3-mysqldb
@@ -1,15 +1,11 @@
 conntrack
 curl
-dnsmasq-base
-dnsmasq-utils # for dhcp_release
 ebtables
-gawk
 genisoimage # required for config_drive
 iptables
 iputils-arping
 kpartx
 libjs-jquery-tablesorter # Needed for coverage html reports
-libmysqlclient-dev
 libvirt-clients # NOPRIME
 libvirt-daemon-system # NOPRIME
 libvirt-dev # NOPRIME
@@ -2,5 +2,6 @@ curl
 liberasurecode-dev
 make
 memcached
+rsync
 sqlite3
 xfsprogs
@@ -1,3 +0,0 @@
-enable-tftp
-tftp-root=/tftpboot
-dhcp-boot=pxelinux.0
@@ -1,4 +1,4 @@
-dn: olcDatabase={${LDAP_OLCDB_NUMBER}}hdb,cn=config
+dn: olcDatabase={${LDAP_OLCDB_NUMBER}}${LDAP_OLCDB_TYPE},cn=config
 changetype: modify
 replace: olcSuffix
 olcSuffix: ${BASE_DN}
@@ -0,0 +1,16 @@
+[Unit]
+Description=Activate LVM backing file %BACKING_FILE%
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+ExecStart=/sbin/losetup --find --show %DIRECTIO% %BACKING_FILE%
+ExecStop=/bin/sh -c '/sbin/losetup -d $$(/sbin/losetup --associated %BACKING_FILE% -O NAME -n)'
+RemainAfterExit=yes
+Type=oneshot
+
+[Install]
+WantedBy=local-fs.target
+Also=systemd-udev-settle.service
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import os.path
+import json
+
+server_address = "/tmp/openstack.sock"
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+try:
+    sock.connect(server_address)
+except socket.error as msg:
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+
+    char = sock.recv(1)
+    if len(char) == 0:
+        print("Unexpected end of file", file=sys.stderr)
+        sys.exit(1)
+
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+        if len(char) == 0:
+            print("Unexpected end of file", file=sys.stderr)
+            sys.exit(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+try:
+    env = {}
+    passenv = ["CINDER_VERSION",
+               "OS_AUTH_URL",
+               "OS_NO_CACHE",
+               "OS_PASSWORD",
+               "OS_PROJECT_NAME",
+               "OS_REGION_NAME",
+               "OS_TENANT_NAME",
+               "OS_USERNAME",
+               "OS_VOLUME_API_VERSION",
+               "OS_CLOUD"]
+    for name in passenv:
+        if name in os.environ:
+            env[name] = os.environ[name]
+
+    cmd = {
+        "app": os.path.basename(sys.argv[0]),
+        "env": env,
+        "argv": sys.argv[1:]
+    }
+    try:
+        image_idx = sys.argv.index('image')
+        create_idx = sys.argv.index('create')
+        missing_file = image_idx < create_idx and \
+                not any(x.startswith('--file') for x in sys.argv)
+    except ValueError:
+        missing_file = False
+
+    if missing_file:
+        # This means we were called with an image create command, but were
+        # not provided a --file option. That likely means we're being passed
+        # the image data to stdin, which won't work because we do not proxy
+        # stdin to the server. So, we just reject the operation and ask the
+        # caller to provide the file with --file instead.
+        # We've already connected to the server, we need to send it some dummy
+        # data so it doesn't wait forever.
+        send(sock, {})
+        print('Image create without --file is not allowed in server mode',
+              file=sys.stderr)
+        sys.exit(1)
+    else:
+        send(sock, cmd)
+
+    doc = recv(sock)
+    if doc["stdout"] != b'':
+        print(doc["stdout"], end='')
+    if doc["stderr"] != b'':
+        print(doc["stderr"], file=sys.stderr)
+    sys.exit(doc["status"])
+finally:
+    sock.close()
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import json
+
+from openstackclient import shell as osc_shell
+from io import StringIO
+
+server_address = "/tmp/openstack.sock"
+
+try:
+    os.unlink(server_address)
+except OSError:
+    if os.path.exists(server_address):
+        raise
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+print('starting up on %s' % server_address, file=sys.stderr)
+sock.bind(server_address)
+
+# Listen for incoming connections
+sock.listen(1)
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+    char = sock.recv(1)
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+while True:
+    csock, client_address = sock.accept()
+    try:
+        doc = recv(csock)
+
+        print("%s %s" % (doc["app"], doc["argv"]), file=sys.stderr)
+        oldenv = {}
+        for name in doc["env"].keys():
+            oldenv[name] = os.environ.get(name, None)
+            os.environ[name] = doc["env"][name]
+
+        try:
+            old_stdout = sys.stdout
+            old_stderr = sys.stderr
+            my_stdout = sys.stdout = StringIO()
+            my_stderr = sys.stderr = StringIO()
+
+            class Exit(BaseException):
+                def __init__(self, status):
+                    self.status = status
+
+            def noexit(stat):
+                raise Exit(stat)
+
+            sys.exit = noexit
+
+            if doc["app"] == "openstack":
+                sh = osc_shell.OpenStackShell()
+                ret = sh.run(doc["argv"])
+            else:
+                print("Unknown application %s" % doc["app"], file=sys.stderr)
+                ret = 1
+        except Exit as e:
+            ret = e.status
+        finally:
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+
+            for name in oldenv.keys():
+                if oldenv[name] is None:
+                    del os.environ[name]
+                else:
+                    os.environ[name] = oldenv[name]
+
+        send(csock, {
+            "stdout": my_stdout.getvalue(),
+            "stderr": my_stderr.getvalue(),
+            "status": ret,
+        })
+
+    except BaseException as e:
+        print(e, file=sys.stderr)
+    finally:
+        csock.close()
@@ -1 +0,0 @@
-dnsmasq
@@ -1,3 +0,0 @@
-ceph    # NOPRIME
-lsb
-xfsprogs
@@ -1,3 +0,0 @@
-lvm2
-qemu-tools
-tgt # NOPRIME
@@ -1 +0,0 @@
-dstat
@@ -1,33 +0,0 @@
-apache2
-apache2-devel
-bc
-ca-certificates-mozilla
-curl
-gcc
-gcc-c++
-git-core
-graphviz # docs
-iputils
-libffi-devel  # pyOpenSSL
-libjpeg8-devel # Pillow 3.0.0
-libopenssl-devel # to rebuild pyOpenSSL if needed
-libxslt-devel  # lxml
-lsof # useful when debugging
-make
-net-tools
-openssh
-openssl
-pcre-devel # python-pcre
-postgresql-devel  # psycopg2
-psmisc
-python-cmd2 # dist:opensuse-12.3
-python-devel  # pyOpenSSL
-python-xml
-systemd-devel # for systemd-python
-tar
-tcpdump
-unzip
-util-linux
-wget
-which
-zlib-devel
@@ -1,2 +0,0 @@
-apache2-mod_wsgi  # NOPRIME
-apache2  # NOPRIME
@@ -1,4 +0,0 @@
-cyrus-sasl-devel
-memcached
-openldap2-devel
-sqlite3
@@ -1,3 +0,0 @@
-openldap2
-openldap2-client
-python-ldap
@@ -1 +0,0 @@
-python-dateutil
@@ -1,10 +0,0 @@
-cdrkit-cdrtools-compat # dist:sle12
-cryptsetup
-dosfstools
-libosinfo
-lvm2
-mkisofs # not:sle12
-open-iscsi
-sg3_utils
-# Stuff for diablo volumes
-sysfsutils
@@ -1 +0,0 @@
-ipset
@@ -1,12 +0,0 @@
-acl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
-ebtables
-haproxy # to serve as metadata proxy inside router/dhcp namespaces
-iptables
-iputils
-rabbitmq-server # NOPRIME
-radvd # NOPRIME
-sqlite3
-sudo
-vlan
@@ -1,2 +0,0 @@
-conntrack-tools
-keepalived
@@ -1,24 +0,0 @@
-cdrkit-cdrtools-compat # dist:sle12
-conntrack-tools
-curl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
-ebtables
-gawk
-iptables
-iputils
-kpartx
-kvm # NOPRIME
-libvirt # NOPRIME
-libvirt-python # NOPRIME
-# mkisofs is required for config_drive
-mkisofs # not:sle12
-parted
-polkit
-# qemu as fallback if kvm cannot be used
-qemu # NOPRIME
-rabbitmq-server # NOPRIME
-socat
-sqlite3
-sudo
-vlan
@@ -1,3 +0,0 @@
-
-openvswitch
-openvswitch-switch
@@ -1,2 +0,0 @@
-lsscsi
-open-iscsi
@@ -1 +0,0 @@
-neutron-agent
@@ -1 +0,0 @@
-neutron-l3
@@ -1,6 +0,0 @@
-curl
-liberasurecode-devel
-memcached
-sqlite3
-xfsprogs
-xinetd
@@ -1,3 +1,3 @@
 ceph    # NOPRIME
-redhat-lsb-core
+redhat-lsb-core # not:rhel9,openEuler-22.03
 xfsprogs
@@ -1,13 +1,16 @@
 bc
 curl
 dbus
+gawk
 gcc
 gcc-c++
 gettext  # used for compiling message catalogs
 git-core
+glibc-langpack-en # dist:rhel9
 graphviz # needed only for docs
 httpd
 httpd-devel
+iptables-nft # dist:rhel9
 iptables-services
 java-1.8.0-openjdk-headless
 libffi-devel
@@ -15,6 +18,7 @@ libjpeg-turbo-devel # Pillow 3.0.0
 libxml2-devel # lxml
 libxslt-devel # lxml
 libyaml-devel
+mod_ssl # required for tls-proxy on centos 9 stream computes
 net-tools
 openssh-server
 openssl
@@ -24,9 +28,9 @@ pkgconfig
 postgresql-devel  # psycopg2
 psmisc
 python3-devel
-python3-pip
-redhat-rpm-config # missing dep for gcc hardening flags, see rhbz#1217376
-systemd-devel # for systemd-python
+python3-pip # not:openEuler-22.03
+python3-systemd
+redhat-rpm-config # not:openEuler-22.03  missing dep for gcc hardening flags, see rhbz#1217376
 tar
 tcpdump
 unzip
@@ -1,9 +1,10 @@
 cryptsetup
 dosfstools
-genisoimage
+genisoimage # not:rhel9
 iscsi-initiator-utils
 libosinfo
 lvm2
 sg3_utils
 # Stuff for diablo volumes
 sysfsutils
+xorriso # not:rhel8
@@ -1,16 +1,14 @@
 conntrack-tools
 curl
-dnsmasq # for q-dhcp
-dnsmasq-utils # for dhcp_release
 ebtables
-gawk
-genisoimage # required for config_drive
+genisoimage # not:rhel9 required for config_drive
 iptables
 iputils
-kernel-modules
+kernel-modules # not:openEuler-22.03
 kpartx
 parted
 polkit
 rabbitmq-server # NOPRIME
 sqlite
 sudo
+xorriso # not:rhel8
@@ -4,4 +4,4 @@ memcached
 rsync-daemon
 sqlite
 xfsprogs
-xinetd
+xinetd # not:f36,rhel9
@@ -21,6 +21,7 @@ source ${FUNC_DIR}/inc/ini-config
 source ${FUNC_DIR}/inc/meta-config
 source ${FUNC_DIR}/inc/python
 source ${FUNC_DIR}/inc/rootwrap
+source ${FUNC_DIR}/inc/async

 # Save trace setting
 _XTRACE_FUNCTIONS=$(set +o | grep xtrace)
@@ -117,7 +118,7 @@ function _upload_image {
        useimport="--import"
    fi

-    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties < "${image}"
+    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties --file $(readlink -f "${image}")
 }

 # Retrieve an image from a URL and upload into Glance.
@@ -132,17 +133,28 @@ function upload_image {

    local image image_fname image_name

+    local max_attempts=5
+
    # Create a directory for the downloaded image tarballs.
    mkdir -p $FILES/images
    image_fname=`basename "$image_url"`
    if [[ $image_url != file* ]]; then
        # Downloads the image (uec ami+akistyle), then extracts it.
        if [[ ! -f $FILES/$image_fname || "$(stat -c "%s" $FILES/$image_fname)" = "0" ]]; then
-            wget --progress=dot:giga -c $image_url -O $FILES/$image_fname
-            if [[ $? -ne 0 ]]; then
-                echo "Not found: $image_url"
-                return
-            fi
+            for attempt in `seq $max_attempts`; do
+                local rc=0
+                wget --progress=dot:giga -c $image_url -O $FILES/$image_fname || rc=$?
+                if [[ $rc -ne 0 ]]; then
+                    if [[ "$attempt" -eq "$max_attempts" ]]; then
+                        echo "Not found: $image_url"
+                        return
+                    fi
+                    echo "Download failed, retrying in $attempt second, attempt: $attempt"
+                    sleep $attempt
+                else
+                    break
+                fi
+            done
        fi
        image="$FILES/${image_fname}"
    else
@@ -279,31 +291,6 @@ function upload_image {
        return
    fi

-    # XenServer-vhd-ovf-format images are provided as .vhd.tgz
-    # and should not be decompressed prior to loading
-    if [[ "$image_url" =~ '.vhd.tgz' ]]; then
-        image_name="${image_fname%.vhd.tgz}"
-        local force_vm_mode=""
-        if [[ "$image_name" =~ 'cirros' ]]; then
-            # Cirros VHD image currently only boots in PV mode.
-            # Nova defaults to PV for all VHD images, but
-            # the glance setting is needed for booting
-            # directly from volume.
-            force_vm_mode="vm_mode=xen"
-        fi
-        _upload_image "$image_name" ovf vhd "$image" $force_vm_mode
-        return
-    fi
-
-    # .xen-raw.tgz suggests a Xen capable raw image inside a tgz.
-    # and should not be decompressed prior to loading.
-    # Setting metadata, so PV mode is used.
-    if [[ "$image_url" =~ '.xen-raw.tgz' ]]; then
-        image_name="${image_fname%.xen-raw.tgz}"
-        _upload_image "$image_name" tgz raw "$image" vm_mode=xen
-        return
-    fi
-
    if [[ "$image_url" =~ '.hds' ]]; then
        image_name="${image_fname%.hds}"
        vm_mode=${image_name##*-}
@@ -438,10 +425,10 @@ function upload_image {
        # kernel for use when uploading the root filesystem.
        local kernel_id="" ramdisk_id="";
        if [ -n "$kernel" ]; then
-            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki < "$kernel" | grep ' id ' | get_field 2)
+            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki --file $(readlink -f "$kernel") -f value -c id)
        fi
        if [ -n "$ramdisk" ]; then
-            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari < "$ramdisk" | grep ' id ' | get_field 2)
+            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari --file $(readlink -f "$ramdisk") -f value -c id)
        fi
        _upload_image "${image_name%.img}" ami ami "$image" ${kernel_id:+ kernel_id=$kernel_id} ${ramdisk_id:+ ramdisk_id=$ramdisk_id} $img_property
    fi
@@ -707,6 +694,8 @@ function setup_colorized_logging {
    iniset $conf_file DEFAULT logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_debug_format_suffix "[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s [01;35m%(instance)s[00m"
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_systemd_logging {
@@ -728,6 +717,9 @@ function setup_systemd_logging {
    iniset $conf_file DEFAULT logging_context_format_string "%(color)s%(levelname)s %(name)s [[01;36m%(global_request_id)s %(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_default_format_string "%(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "ERROR %(name)s [01;35m%(instance)s[00m"
+
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_standard_logging_identity {
@@ -751,23 +743,22 @@ if ! function_exists echo_nolog; then
 fi


-# create_disk - Create backing disk
+# create_disk - Create, configure, and mount a backing disk
 function create_disk {
    local node_number
    local disk_image=${1}
    local storage_data_dir=${2}
    local loopback_disk_size=${3}
+    local key

-    # Create a loopback disk and format it to XFS.
-    if [[ -e ${disk_image} ]]; then
-        if egrep -q ${storage_data_dir} /proc/mounts; then
-            sudo umount ${storage_data_dir}
-            sudo rm -f ${disk_image}
-        fi
-    fi
+    key=$(echo $disk_image | sed 's#/.##')
+    key="devstack-$key"

-    sudo mkdir -p ${storage_data_dir}/drives/images
+    destroy_disk $disk_image $storage_data_dir

+    # Create an empty file of the correct size (and ensure the
+    # directory structure up to that path exists)
+    sudo mkdir -p $(dirname ${disk_image})
    sudo truncate -s ${loopback_disk_size} ${disk_image}

    # Make a fresh XFS filesystem. Use bigger inodes so xattr can fit in
@@ -777,11 +768,31 @@ function create_disk {
    # Swift and Ceph.
    sudo mkfs.xfs -f -i size=1024 ${disk_image}

-    # Mount the disk with mount options to make it as efficient as possible
-    if ! egrep -q ${storage_data_dir} /proc/mounts; then
-        sudo mount -t xfs -o loop,noatime,nodiratime,logbufs=8  \
-            ${disk_image} ${storage_data_dir}
+    # Install a new loopback fstab entry for this disk image, and mount it
+    echo "$disk_image $storage_data_dir xfs loop,noatime,nodiratime,logbufs=8,comment=$key 0 0" | sudo tee -a /etc/fstab
+    sudo mkdir -p $storage_data_dir
+    sudo mount -v $storage_data_dir
+}
+
+# Unmount, de-configure, and destroy a backing disk
+function destroy_disk {
+    local disk_image=$1
+    local storage_data_dir=$2
+    local key
+
+    key=$(echo $disk_image | sed 's#/.##')
+    key="devstack-$key"
+
+    # Unmount the target, if mounted
+    if egrep -q $storage_data_dir /proc/mounts; then
+        sudo umount $storage_data_dir
    fi
+
+    # Clear any fstab rules
+    sudo sed -i '/.*comment=$key.*/ d' /etc/fstab
+
+    # Delete the file
+    sudo rm -f $disk_image
 }


@@ -49,7 +49,7 @@ KILL_PATH="$(which kill)"
 STACK_ENV_VARS="BASE_SQL_CONN DATA_DIR DEST ENABLED_SERVICES HOST_IP \
    KEYSTONE_SERVICE_URI \
    LOGFILE OS_CACERT SERVICE_HOST STACK_USER TLS_IP \
-    HOST_IPV6 SERVICE_IP_VERSION"
+    HOST_IPV6 SERVICE_IP_VERSION TUNNEL_ENDPOINT_IP TUNNEL_IP_VERSION"


 # Saves significant environment variables to .stackenv for later use
@@ -85,7 +85,7 @@ function write_clouds_yaml {
    if [ -f "$SSL_BUNDLE_FILE" ]; then
        CA_CERT_ARG="--os-cacert $SSL_BUNDLE_FILE"
    fi
-    # demo -> devstack
+    # devstack: user with the member role on demo project
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack \
@@ -96,18 +96,7 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-project-name demo

-    # alt_demo -> devstack-alt
-    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
-        --file $CLOUDS_YAML \
-        --os-cloud devstack-alt \
-        --os-region-name $REGION_NAME \
-        $CA_CERT_ARG \
-        --os-auth-url $KEYSTONE_SERVICE_URI \
-        --os-username alt_demo \
-        --os-password $ADMIN_PASSWORD \
-        --os-project-name alt_demo
-
-    # admin -> devstack-admin
+    # devstack-admin: user with the admin role on the admin project
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack-admin \
@@ -118,7 +107,62 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-project-name admin

-    # admin with a system-scoped token -> devstack-system
+    # devstack-admin-demo: user with the admin role on the demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-admin-demo \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username admin \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name demo
+
+    # devstack-alt: user with the member role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-alt-member: user with the member role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt-member \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo_member \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-alt-reader: user with the reader role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-reader: user with the reader role on demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username demo_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name demo
+
+    # devstack-system-admin: user with the admin role on the system
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack-system-admin \
@@ -129,6 +173,28 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-system-scope all

+    # devstack-system-member: user with the member role on the system
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-system-member \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username system_member \
+        --os-password $ADMIN_PASSWORD \
+        --os-system-scope all
+
+    # devstack-system-reader: user with the reader role on the system
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-system-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username system_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-system-scope all
+
    cat >> $CLOUDS_YAML <<EOF
 functional:
  image_name: $DEFAULT_IMAGE_NAME
@@ -170,6 +236,27 @@ function trueorfalse {
    $xtrace
 }

+# bool_to_int <True|False>
+#
+# Convert True|False to int 1 or 0
+# This function can be used to convert the output of trueorfalse
+# to an int follow c conventions where false is 0 and 1 it true.
+function bool_to_int {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    if [ -z $1 ]; then
+        die $LINENO "Bool value required"
+    fi
+    if [[ $1 == "True" ]] ; then
+        echo '1'
+    else
+        echo '0'
+    fi
+    $xtrace
+}
+
+
 function isset {
    [[ -v "$1" ]]
 }
@@ -314,9 +401,9 @@ function warn {
 # such as "install_package" further abstract things in better ways.
 #
 # ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
-# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
+# ``os_RELEASE`` - major release: ``22.04`` (Ubuntu), ``23`` (Fedora)
 # ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
-# ``os_CODENAME`` - vendor's codename for release: ``xenial``
+# ``os_CODENAME`` - vendor's codename for release: ``jammy``

 declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME

@@ -333,7 +420,7 @@ function _ensure_lsb_release {
    elif [[ -x $(command -v zypper 2>/dev/null) ]]; then
        sudo zypper -n install lsb-release
    elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
-        sudo dnf install -y redhat-lsb-core
+        sudo dnf install -y redhat-lsb-core || sudo dnf install -y openeuler-lsb
    else
        die $LINENO "Unable to find or auto-install lsb_release"
    fi
@@ -346,14 +433,24 @@ function _ensure_lsb_release {
 #  - os_VENDOR
 #  - os_PACKAGE
 function GetOSVersion {
-    # We only support distros that provide a sane lsb_release
-    _ensure_lsb_release
+    # CentOS Stream 9 and RHEL 9 do not provide lsb_release
+    source /etc/os-release
+    if [[ "${ID}${VERSION}" == "centos9" ]] || [[ "${ID}${VERSION}" =~ "rhel9" ]]; then
+        os_RELEASE=${VERSION_ID}
+        os_CODENAME="n/a"
+        os_VENDOR=$(echo $NAME | tr -d '[:space:]')
+    elif [[ "${ID}${VERSION}" =~ "rocky9" ]]; then
+        os_VENDOR="Rocky"
+        os_RELEASE=${VERSION_ID}
+    else
+        _ensure_lsb_release

-    os_RELEASE=$(lsb_release -r -s)
-    os_CODENAME=$(lsb_release -c -s)
-    os_VENDOR=$(lsb_release -i -s)
+        os_RELEASE=$(lsb_release -r -s)
+        os_CODENAME=$(lsb_release -c -s)
+        os_VENDOR=$(lsb_release -i -s)
+    fi

-    if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
+    if [[ $os_VENDOR =~ (Debian|Ubuntu) ]]; then
        os_PACKAGE="deb"
    else
        os_PACKAGE="rpm"
@@ -371,34 +468,25 @@ declare -g DISTRO

 function GetDistro {
    GetOSVersion
-    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
-            "$os_VENDOR" =~ (LinuxMint) ]]; then
-        # 'Everyone' refers to Ubuntu / Debian / Mint releases by
+    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
+        # 'Everyone' refers to Ubuntu / Debian releases by
        # the code name adjective
        DISTRO=$os_CODENAME
    elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
        # For Fedora, just use 'f' and the release
        DISTRO="f$os_RELEASE"
-    elif is_opensuse; then
-        DISTRO="opensuse-$os_RELEASE"
-        # Tumbleweed uses "n/a" as a codename, and the release is a datestring
-        # like 20180218, so not very useful. Leap however uses a release
-        # with a "dot", so for example 15.0
-        [ "$os_CODENAME" = "n/a" -a "$os_RELEASE" = "${os_RELEASE/\./}" ] && \
-            DISTRO="opensuse-tumbleweed"
-    elif is_suse_linux_enterprise; then
-        # just use major release
-        DISTRO="sle${os_RELEASE%.*}"
    elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
        "$os_VENDOR" =~ (CentOS) || \
+        "$os_VENDOR" =~ (AlmaLinux) || \
        "$os_VENDOR" =~ (Scientific) || \
        "$os_VENDOR" =~ (OracleServer) || \
+        "$os_VENDOR" =~ (Rocky) || \
        "$os_VENDOR" =~ (Virtuozzo) ]]; then
        # Drop the . release as we assume it's compatible
        # XXX re-evaluate when we get RHEL10
        DISTRO="rhel${os_RELEASE::1}"
-    elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
-        DISTRO="xs${os_RELEASE%.*}"
+    elif [[ "$os_VENDOR" =~ (openEuler) ]]; then
+        DISTRO="openEuler-$os_RELEASE"
    else
        # We can't make a good choice here.  Setting a sensible DISTRO
        # is part of the problem, but not the major issue -- we really
@@ -442,7 +530,7 @@ function is_oraclelinux {


 # Determine if current distribution is a Fedora-based distribution
-# (Fedora, RHEL, CentOS, etc).
+# (Fedora, RHEL, CentOS, Rocky, etc).
 # is_fedora
 function is_fedora {
    if [[ -z "$os_VENDOR" ]]; then
@@ -450,41 +538,14 @@ function is_fedora {
    fi

    [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
+        [ "$os_VENDOR" = "openEuler" ] || \
        [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
        [ "$os_VENDOR" = "RedHatEnterprise" ] || \
-        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \
-        [ "$os_VENDOR" = "Virtuozzo" ]
-}
-
-
-# Determine if current distribution is a SUSE-based distribution
-# (openSUSE, SLE).
-# is_suse
-function is_suse {
-    is_opensuse || is_suse_linux_enterprise
-}
-
-
-# Determine if current distribution is an openSUSE distribution
-# is_opensuse
-function is_opensuse {
-    if [[ -z "$os_VENDOR" ]]; then
-        GetOSVersion
-    fi
-
-    [[ "$os_VENDOR" =~ (openSUSE) ]]
-}
-
-
-# Determine if current distribution is a SUSE Linux Enterprise (SLE)
-# distribution
-# is_suse_linux_enterprise
-function is_suse_linux_enterprise {
-    if [[ -z "$os_VENDOR" ]]; then
-        GetOSVersion
-    fi
-
-    [[ "$os_VENDOR" =~ (^SUSE) ]]
+        [ "$os_VENDOR" = "RedHatEnterpriseLinux" ] || \
+        [ "$os_VENDOR" = "Rocky" ] || \
+        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \
+        [ "$os_VENDOR" = "AlmaLinux" ] || \
+        [ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ]
 }


@@ -498,7 +559,14 @@ function is_ubuntu {
    [ "$os_PACKAGE" = "deb" ]
 }

-
+# Determine if current distribution is an openEuler distribution
+# is_openeuler
+function is_openeuler {
+    if [[ -z "$os_PACKAGE" ]]; then
+        GetOSVersion
+    fi
+    [ "$os_VENDOR" = "openEuler" ]
+}
 # Git Functions
 # =============

@@ -549,7 +617,7 @@ function git_clone {
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to 'required-projects' in the job definition."
-                die $LINENO "Cloning not allowed in this configuration"
+                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
            git_timed clone $git_clone_flags $git_remote $git_dest
        fi
@@ -561,10 +629,12 @@ function git_clone {
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to the \$PROJECTS variable in the job definition."
-                die $LINENO "Cloning not allowed in this configuration"
+                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
-            # '--branch' can also take tags
-            git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref
+            git_timed clone --no-checkout $git_clone_flags $git_remote $git_dest
+            cd $git_dest
+            git_timed fetch $git_clone_flags origin $git_ref
+            git_timed checkout FETCH_HEAD
        elif [[ "$RECLONE" = "True" ]]; then
            # if it does exist then simulate what clone does if asked to RECLONE
            cd $git_dest
@@ -574,7 +644,7 @@ function git_clone {
            # remove the existing ignored files (like pyc) as they cause breakage
            # (due to the py files having older timestamps than our pyc, so python
            # thinks the pyc files are correct using them)
-            find $git_dest -name '*.pyc' -delete
+            sudo find $git_dest -name '*.pyc' -delete

            # handle git_ref accordingly to type (tag, branch)
            if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
@@ -590,6 +660,18 @@ function git_clone {
        fi
    fi

+    # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
+    # about how we clone and work with repos.  Mark them safe globally
+    # as a work-around.
+    #
+    # NOTE(danms): On bionic (and likely others) git-config may write
+    # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+    # writes these changes to /etc/gitconfig which is more
+    # discoverable anyway.
+    #
+    # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
+    sudo git config --system --add safe.directory ${git_dest}
+
    # print out the results so we know what change was used in the logs
    cd $git_dest
    git show --oneline | head -1
@@ -689,7 +771,7 @@ function get_default_host_ip {
    if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
        host_ip=""
        # Find the interface used for the default route
-        host_ip_iface=${host_ip_iface:-$(ip -f $af route | awk '/default/ {print $5}' | head -1)}
+        host_ip_iface=${host_ip_iface:-$(ip -f $af route list match default table all | grep via | awk '/default/ {print $5}' | head -1)}
        local host_ips
        host_ips=$(LC_ALL=C ip -f $af addr show ${host_ip_iface} | sed /temporary/d |awk /$af'/ {split($2,parts,"/");  print parts[1]}')
        local ip
@@ -792,14 +874,9 @@ function policy_add {
 # Usage: get_or_create_domain <name> <description>
 function get_or_create_domain {
    local domain_id
-    # Gets domain id
    domain_id=$(
-        # Gets domain id
-        openstack domain show $1 \
-            -f value -c id 2>/dev/null ||
-        # Creates new domain
-        openstack domain create $1 \
-            --description "$2" \
+        openstack --os-cloud devstack-system-admin domain create $1 \
+            --description "$2" --or-show \
            -f value -c id
    )
    echo $domain_id
@@ -813,7 +890,7 @@ function get_or_create_group {
    # Gets group id
    group_id=$(
        # Creates new group with --or-show
-        openstack group create $1 \
+        openstack --os-cloud devstack-system-admin group create $1 \
            --domain $2 --description "$desc" --or-show \
            -f value -c id
    )
@@ -832,7 +909,7 @@ function get_or_create_user {
    # Gets user id
    user_id=$(
        # Creates new user with --or-show
-        openstack user create \
+        openstack --os-cloud devstack-system-admin user create \
            $1 \
            --password "$2" \
            --domain=$3 \
@@ -849,7 +926,7 @@ function get_or_create_project {
    local project_id
    project_id=$(
        # Creates new project with --or-show
-        openstack project create $1 \
+        openstack --os-cloud devstack-system-admin project create $1 \
            --domain=$2 \
            --or-show -f value -c id
    )
@@ -862,7 +939,7 @@ function get_or_create_role {
    local role_id
    role_id=$(
        # Creates role with --or-show
-        openstack role create $1 \
+        openstack --os-cloud devstack-system-admin role create $1 \
            --or-show -f value -c id
    )
    echo $role_id
@@ -888,29 +965,22 @@ function _get_domain_args {
 # Usage: get_or_add_user_project_role <role> <user> <project> [<user_domain> <project_domain>]
 function get_or_add_user_project_role {
    local user_role_id
+    local domain_args

    domain_args=$(_get_domain_args $4 $5)

-    # Gets user role id
-    user_role_id=$(openstack role assignment list \
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack --os-cloud devstack-system-admin role add $1 \
+        --user $2 \
+        --project $3 \
+        $domain_args
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --project $3 \
        $domain_args \
-        | grep '^|\s[a-f0-9]\+' | get_field 1)
-    if [[ -z "$user_role_id" ]]; then
-        # Adds role to user and get it
-        openstack role add $1 \
-            --user $2 \
-            --project $3 \
-            $domain_args
-        user_role_id=$(openstack role assignment list \
-            --role $1 \
-            --user $2 \
-            --project $3 \
-            $domain_args \
-            | grep '^|\s[a-f0-9]\+' | get_field 1)
-    fi
+        -c Role -f value)
    echo $user_role_id
 }

@@ -918,22 +988,48 @@ function get_or_add_user_project_role {
 # Usage: get_or_add_user_domain_role <role> <user> <domain>
 function get_or_add_user_domain_role {
    local user_role_id
-    # Gets user role id
-    user_role_id=$(openstack role assignment list \
+
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack --os-cloud devstack-system-admin role add $1 \
+        --user $2 \
+        --domain $3
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --domain $3 \
-        | grep '^|\s[a-f0-9]\+' | get_field 1)
+        -c Role -f value)
+
+    echo $user_role_id
+}
+
+# Gets or adds user role to system
+# Usage: get_or_add_user_system_role <role> <user> <system> [<user_domain>]
+function get_or_add_user_system_role {
+    local user_role_id
+    local domain_args
+
+    domain_args=$(_get_domain_args $4)
+
+    # Gets user role id
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
+        --role $1 \
+        --user $2 \
+        --system $3 \
+        $domain_args \
+        -f value -c Role)
    if [[ -z "$user_role_id" ]]; then
        # Adds role to user and get it
-        openstack role add $1 \
+        openstack --os-cloud devstack-system-admin role add $1 \
            --user $2 \
-            --domain $3
-        user_role_id=$(openstack role assignment list \
+            --system $3 \
+            $domain_args
+        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
            --role $1 \
            --user $2 \
-            --domain $3 \
-            | grep '^|\s[a-f0-9]\+' | get_field 1)
+            --system $3 \
+            $domain_args \
+            -f value -c Role)
    fi
    echo $user_role_id
 }
@@ -942,23 +1038,18 @@ function get_or_add_user_domain_role {
 # Usage: get_or_add_group_project_role <role> <group> <project>
 function get_or_add_group_project_role {
    local group_role_id
-    # Gets group role id
-    group_role_id=$(openstack role assignment list \
+
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack role add $1 \
+        --group $2 \
+        --project $3
+    group_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --group $2 \
        --project $3 \
-        -f value)
-    if [[ -z "$group_role_id" ]]; then
-        # Adds role to group and get it
-        openstack role add $1 \
-            --group $2 \
-            --project $3
-        group_role_id=$(openstack role assignment list \
-            --role $1 \
-            --group $2 \
-            --project $3 \
-            -f value)
-    fi
+        -f value -c Role)
+
    echo $group_role_id
 }

@@ -969,9 +1060,9 @@ function get_or_create_service {
    # Gets service id
    service_id=$(
        # Gets service id
-        openstack service show $2 -f value -c id 2>/dev/null ||
+        openstack --os-cloud devstack-system-admin service show $2 -f value -c id 2>/dev/null ||
        # Creates new service if not exists
-        openstack service create \
+        openstack --os-cloud devstack-system-admin service create \
            $2 \
            --name $1 \
            --description="$3" \
@@ -984,14 +1075,14 @@ function get_or_create_service {
 # Usage: _get_or_create_endpoint_with_interface <service> <interface> <url> <region>
 function _get_or_create_endpoint_with_interface {
    local endpoint_id
-    endpoint_id=$(openstack endpoint list \
+    endpoint_id=$(openstack --os-cloud devstack-system-admin endpoint list \
        --service $1 \
        --interface $2 \
        --region $4 \
        -c ID -f value)
    if [[ -z "$endpoint_id" ]]; then
        # Creates new endpoint
-        endpoint_id=$(openstack endpoint create \
+        endpoint_id=$(openstack --os-cloud devstack-system-admin endpoint create \
            $1 $2 $3 --region $4 -f value -c id)
    fi

@@ -1025,7 +1116,7 @@ function get_or_create_endpoint {
 # Get a URL from the identity service
 # Usage: get_endpoint_url <service> <interface>
 function get_endpoint_url {
-    echo $(openstack endpoint list \
+    echo $(openstack --os-cloud devstack-system-admin endpoint list \
            --service $1 --interface $2 \
            -c URL -f value)
 }
@@ -1039,6 +1130,17 @@ function is_ironic_hardware {
    return 1
 }

+function is_ironic_enforce_scope {
+    is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]] && return 0
+    return 1
+}
+
+function is_ironic_sharded {
+    # todo(JayF): Support >1 shard with multiple n-cpu instances for each
+    is_service_enabled ironic && [[ "$IRONIC_SHARDS" == "1" ]] && return 0
+    return 1
+}
+

 # Package Functions
 # =================
@@ -1055,8 +1157,6 @@ function _get_package_dir {
        pkg_dir=$base_dir/debs
    elif is_fedora; then
        pkg_dir=$base_dir/rpms
-    elif is_suse; then
-        pkg_dir=$base_dir/rpms-suse
    else
        exit_distro_not_supported "list of packages"
    fi
@@ -1331,8 +1431,6 @@ function real_install_package {
        apt_get install "$@"
    elif is_fedora; then
        yum_install "$@"
-    elif is_suse; then
-        zypper_install "$@"
    else
        exit_distro_not_supported "installing packages"
    fi
@@ -1374,8 +1472,6 @@ function uninstall_package {
        apt_get purge "$@"
    elif is_fedora; then
        sudo dnf remove -y "$@" ||:
-    elif is_suse; then
-        sudo zypper remove -y "$@" ||:
    else
        exit_distro_not_supported "uninstalling packages"
    fi
@@ -1444,6 +1540,7 @@ function write_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local extra=""
    if [[ -n "$group" ]]; then
        extra="Group=$group"
@@ -1452,11 +1549,15 @@ function write_user_unit_file {
    mkdir -p $SYSTEMD_DIR

    iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
    iniset -sudo $unitfile "Service" "User" "$user"
    iniset -sudo $unitfile "Service" "ExecStart" "$command"
    iniset -sudo $unitfile "Service" "KillMode" "process"
    iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
    iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1471,10 +1572,12 @@ function write_uwsgi_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local unitfile="$SYSTEMD_DIR/$service"
    mkdir -p $SYSTEMD_DIR

    iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
    iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
    iniset -sudo $unitfile "Service" "User" "$user"
    iniset -sudo $unitfile "Service" "ExecStart" "$command"
@@ -1485,6 +1588,9 @@ function write_uwsgi_user_unit_file {
    iniset -sudo $unitfile "Service" "NotifyAccess" "all"
    iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"

+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1532,10 +1638,17 @@ function _run_under_systemd {
    local systemd_service="devstack@$service.service"
    local group=$3
    local user=${4:-$STACK_USER}
+    if [[ -z "$user" ]]; then
+        user=$STACK_USER
+    fi
+    local env_vars="$5"
    if [[ "$command" =~ "uwsgi" ]] ; then
-        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        if [[ "$GLOBAL_VENV" == "True" ]] ; then
+            cmd="$cmd --venv $DEVSTACK_VENV"
+        fi
+        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    else
-        write_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    fi

    $SYSTEMCTL enable $systemd_service
@@ -1556,18 +1669,20 @@ function is_running {
 # If the command includes shell metachatacters (;<>*) it must be run using a shell
 # If an optional group is provided sg will be used to run the
 # command as that group.
-# run_process service "command-line" [group] [user]
+# run_process service "command-line" [group] [user] [env_vars]
+# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2"
 function run_process {
    local service=$1
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"

    local name=$service

    time_start "run_process"
    if is_service_enabled $service; then
-        _run_under_systemd "$name" "$command" "$group" "$user"
+        _run_under_systemd "$name" "$command" "$group" "$user" "$env_vars"
    fi
    time_stop "run_process"
 }
@@ -1609,10 +1724,6 @@ function service_check {
 }


-function tail_log {
-    deprecated "With the removal of screen support, tail_log is deprecated and will be removed after Queens"
-}
-
 # Plugin Functions
 # =================

@@ -2327,6 +2438,11 @@ function time_stop {
    _TIME_TOTAL[$name]=$(($total + $elapsed_time))
 }

+function install_openstack_cli_server {
+    export PATH=$TOP_DIR/files/openstack-cli-server:$PATH
+    run_process openstack-cli-server "$PYTHON $TOP_DIR/files/openstack-cli-server/openstack-cli-server"
+}
+
 function oscwrap {
    local xtrace
    xtrace=$(set +o | grep xtrace)
@@ -2422,6 +2538,11 @@ function clean_pyc_files {
    fi
 }

+function is_fips_enabled {
+    fips=`cat /proc/sys/crypto/fips_enabled`
+    [ "$fips" == "1" ]
+}
+
 # Restore xtrace
 $_XTRACE_FUNCTIONS_COMMON

@@ -0,0 +1,256 @@
+#!/bin/bash
+#
+# Symbolic asynchronous tasks for devstack
+#
+# Usage:
+#
+#  async_runfunc my_shell_func foo bar baz
+#
+#  ... do other stuff ...
+#
+#  async_wait my_shell_func
+#
+
+DEVSTACK_PARALLEL=$(trueorfalse True DEVSTACK_PARALLEL)
+_ASYNC_BG_TIME=0
+
+# Keep track of how much total time was spent in background tasks
+# Takes a job runtime in ms.
+function _async_incr_bg_time {
+    local elapsed_ms="$1"
+    _ASYNC_BG_TIME=$(($_ASYNC_BG_TIME + $elapsed_ms))
+}
+
+# Get the PID of a named future to wait on
+function async_pidof {
+    local name="$1"
+    local inifile="${DEST}/async/${name}.ini"
+
+    if [ -f "$inifile" ]; then
+        iniget $inifile job pid
+    else
+        echo 'UNKNOWN'
+        return 1
+    fi
+}
+
+# Log a message about a job. If the message contains "%command" then the
+# full command line of the job will be substituted in the output
+function async_log {
+    local name="$1"
+    shift
+    local message="$*"
+    local inifile=${DEST}/async/${name}.ini
+    local pid
+    local command
+
+    pid=$(iniget $inifile job pid)
+    command=$(iniget $inifile job command | tr '#' '-')
+    message=$(echo "$message" | sed "s#%command#$command#g")
+
+    echo "[$BASHPID Async ${name}:${pid}]: $message"
+}
+
+# Inner function that actually runs the requested task. We wrap it like this
+# just so we can emit a finish message as soon as the work is done, to make
+# it easier to find the tracking just before an error.
+function async_inner {
+    local name="$1"
+    local rc
+    local fifo="${DEST}/async/${name}.fifo"
+    shift
+    set -o xtrace
+    if $* >${DEST}/async/${name}.log 2>&1; then
+        rc=0
+        set +o xtrace
+        async_log "$name" "finished successfully"
+    else
+        rc=$?
+        set +o xtrace
+        async_log "$name" "FAILED with rc $rc"
+    fi
+    iniset ${DEST}/async/${name}.ini job end_time $(date "+%s%3N")
+    # Block on the fifo until we are signaled to exit by the main process
+    cat $fifo
+    return $rc
+}
+
+# Run something async. Takes a symbolic name and a list of arguments of
+# what to run. Ideally this would be rarely used and async_runfunc() would
+# be used everywhere for readability.
+#
+# This spawns the work in a background worker, records a "future" to be
+# collected by a later call to async_wait()
+function async_run {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    local name="$1"
+    shift
+    local inifile=${DEST}/async/${name}.ini
+    local fifo=${DEST}/async/${name}.fifo
+
+    touch $inifile
+    iniset $inifile job command "$*"
+    iniset $inifile job start_time $(date +%s%3N)
+
+    if [[ "$DEVSTACK_PARALLEL" = "True" ]]; then
+        mkfifo $fifo
+        async_inner $name $* &
+        iniset $inifile job pid $!
+        async_log "$name" "running: %command"
+        $xtrace
+    else
+        iniset $inifile job pid "self"
+        async_log "$name" "Running synchronously: %command"
+        $xtrace
+        $*
+        return $?
+    fi
+}
+
+# Shortcut for running a shell function async. Uses the function name as the
+# async name.
+function async_runfunc {
+    async_run $1 $*
+}
+
+# Dump some information to help debug a failed wait
+function async_wait_dump {
+    local failpid=$1
+
+    echo "=== Wait failure dump from $BASHPID ==="
+    echo "Processes:"
+    ps -f
+    echo "Waiting jobs:"
+    for name in $(ls ${DEST}/async/*.ini); do
+        echo "Job $name :"
+        cat "$name"
+    done
+    echo "Failed PID status:"
+    sudo cat /proc/$failpid/status
+    sudo cat /proc/$failpid/cmdline
+    echo "=== End wait failure dump ==="
+}
+
+# Wait for an async future to complete. May return immediately if already
+# complete, or of the future has already been waited on (avoid this). May
+# block until the future completes.
+function async_wait {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    local pid rc running inifile runtime fifo
+    rc=0
+    for name in $*; do
+        running=$(ls ${DEST}/async/*.ini 2>/dev/null | wc -l)
+        inifile="${DEST}/async/${name}.ini"
+        fifo="${DEST}/async/${name}.fifo"
+
+        if pid=$(async_pidof "$name"); then
+            async_log "$name" "Waiting for completion of %command" \
+                      "running on PID $pid ($running other jobs running)"
+            time_start async_wait
+            if [[ "$pid" != "self" ]]; then
+                # Signal the child to go ahead and exit since we are about to
+                # wait for it to collect its status.
+                async_log "$name" "Signaling child to exit"
+                echo WAKEUP > $fifo
+                async_log "$name" "Signaled"
+                # Do not actually call wait if we ran synchronously
+                if wait $pid; then
+                    rc=0
+                else
+                    rc=$?
+                fi
+                cat ${DEST}/async/${name}.log
+                rm -f $fifo
+            fi
+            time_stop async_wait
+            local start_time
+            local end_time
+            start_time=$(iniget $inifile job start_time)
+            end_time=$(iniget $inifile job end_time)
+            _async_incr_bg_time $(($end_time - $start_time))
+            runtime=$((($end_time - $start_time) / 1000))
+            async_log "$name" "finished %command with result" \
+                      "$rc in $runtime seconds"
+            rm -f $inifile
+            if [ $rc -ne 0 ]; then
+                async_wait_dump $pid
+                echo Stopping async wait due to error: $*
+                break
+            fi
+        else
+            # This could probably be removed - it is really just here
+            # to help notice if you wait for something by the wrong
+            # name, but it also shows up for things we didn't start
+            # because they were not enabled.
+            echo Not waiting for async task $name that we never started or \
+                 has already been waited for
+        fi
+    done
+
+    $xtrace
+    return $rc
+}
+
+# Check for uncollected futures and wait on them
+function async_cleanup {
+    local name
+
+    if [[ "$DEVSTACK_PARALLEL" != "True" ]]; then
+        return 0
+    fi
+
+    for inifile in $(find ${DEST}/async -name '*.ini'); do
+        name=$(basename $pidfile .ini)
+        echo "WARNING: uncollected async future $name"
+        async_wait $name || true
+    done
+}
+
+# Make sure our async dir is created and clean
+function async_init {
+    local async_dir=${DEST}/async
+
+    # Clean any residue if present from previous runs
+    rm -Rf $async_dir
+
+    # Make sure we have a state directory
+    mkdir -p $async_dir
+}
+
+function async_print_timing {
+    local bg_time_minus_wait
+    local elapsed_time
+    local serial_time
+    local speedup
+
+    if [[ "$DEVSTACK_PARALLEL" != "True" ]]; then
+        return 0
+    fi
+
+    # The logic here is: All the background task time would be
+    # serialized if we did not do them in the background. So we can
+    # add that to the elapsed time for the whole run. However, time we
+    # spend waiting for async things to finish adds to the elapsed
+    # time, but is time where we're not doing anything useful. Thus,
+    # we substract that from the would-be-serialized time.
+
+    bg_time_minus_wait=$((\
+            ($_ASYNC_BG_TIME - ${_TIME_TOTAL[async_wait]}) / 1000))
+    elapsed_time=$(($(date "+%s") - $_TIME_BEGIN))
+    serial_time=$(($elapsed_time + $bg_time_minus_wait))
+
+    echo
+    echo "================="
+    echo " Async summary"
+    echo "================="
+    echo " Time spent in the background minus waits: $bg_time_minus_wait sec"
+    echo " Elapsed time: $elapsed_time sec"
+    echo " Time if we did everything serially: $serial_time sec"
+    echo " Speedup: " $(echo | awk "{print $serial_time / $elapsed_time}")
+}
@@ -189,6 +189,9 @@ function iniset {
    local option=$3
    local value=$4

+    # Escape the ampersand character (&)
+    value=$(echo $value | sed -e 's/&/\\&/g')
+
    if [[ -z $section || -z $option ]]; then
        $xtrace
        return
@@ -7,7 +7,6 @@
 # External functions used:
 # - GetOSVersion
 # - is_fedora
-# - is_suse
 # - safe_chown

 # Save trace setting
@@ -33,6 +32,26 @@ function join_extras {
 # Python Functions
 # ================

+# Setup the global devstack virtualenvs and the associated environment
+# updates.
+function setup_devstack_virtualenv {
+    # We run devstack out of a global virtualenv.
+    if [[ ! -d $DEVSTACK_VENV ]] ; then
+        # Using system site packages to enable nova to use libguestfs.
+        # This package is currently installed via the distro and not
+        # available on pypi.
+        python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
+        pip_install -U pip setuptools
+        #NOTE(rpittau): workaround for simplejson removal in osc
+        #  https://review.opendev.org/c/openstack/python-openstackclient/+/920001
+        pip_install -U simplejson
+    fi
+    if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
+        export PATH="$DEVSTACK_VENV/bin:$PATH"
+        export PYTHON="$DEVSTACK_VENV/bin/python3"
+    fi
+}
+
 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
@@ -61,9 +80,11 @@ function get_python_exec_prefix {
    fi
    $xtrace

-    local PYTHON_PATH=/usr/local/bin
-    is_suse && PYTHON_PATH=/usr/bin
-    echo $PYTHON_PATH
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        echo "$DEVSTACK_VENV/bin"
+    else
+        echo "/usr/local/bin"
+    fi
 }

 # Wrapper for ``pip install`` that only installs versions of libraries
@@ -168,6 +189,14 @@ function pip_install {
    if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
        local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
        local sudo_pip="env"
+    elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
+        # We have to check that the DEVSTACK_VENV exists because early
+        # devstack boostrapping needs to operate in a system context
+        # too bootstrap pip. Once pip is bootstrapped we create the
+        # global venv and can start to use it.
+        local cmd_pip=$DEVSTACK_VENV/bin/pip
+        local sudo_pip="env"
+        echo "Using python $PYTHON3_VERSION to install $package_dir"
    else
        local cmd_pip="python$PYTHON3_VERSION -m pip"
        # See
@@ -186,15 +215,11 @@ function pip_install {

    $xtrace

-    # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
-    # the same behaviour of setuptools before version 25.0.0.
-    # related issue: https://github.com/pypa/pip/issues/3874
    $sudo_pip \
        http_proxy="${http_proxy:-}" \
        https_proxy="${https_proxy:-}" \
        no_proxy="${no_proxy:-}" \
        PIP_FIND_LINKS=$PIP_FIND_LINKS \
-        SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
        $cmd_pip $upgrade \
        $@
    result=$?
@@ -378,12 +403,16 @@ function _setup_package_with_constraints_edit {
    project_dir=$(cd $project_dir && pwd)

    if [ -n "$REQUIREMENTS_DIR" ]; then
-        # Constrain this package to this project directory from here on out.
+        # Remove this package from constraints before we install it.
+        # That way, later installs won't "downgrade" the install from
+        # source we are about to do.
        local name
        name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
+        if [ -z $name ]; then
+            name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
+        fi
        $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
-            $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
-            "$flags file://$project_dir#egg=$name"
+            $REQUIREMENTS_DIR/upper-constraints.txt -- $name
    fi

    setup_package $bindep $project_dir "$flags" $extras
@@ -444,8 +473,11 @@ function setup_package {

    pip_install $flags "$project_dir$extras"
    # ensure that further actions can do things like setup.py sdist
-    if [[ "$flags" == "-e" ]]; then
-        safe_chown -R $STACK_USER $1/*.egg-info
+    if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
+        # egg-info is not created when project have pyproject.toml
+        if [ -d $1/*.egg-info ]; then
+            safe_chown -R $STACK_USER $1/*.egg-info
+        fi
    fi
 }

@@ -465,8 +497,6 @@ function install_python {
 function install_python3 {
    if is_ubuntu; then
        apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
-    elif is_suse; then
-        install_package python3-devel python3-dbm
    elif is_fedora; then
        if [ "$os_VENDOR" = "Fedora" ]; then
            install_package python${PYTHON3_VERSION//.}
@@ -60,6 +60,11 @@ function configure_rootwrap {
    sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf
    sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf

+    # Rely on $PATH set by devstack to determine what is safe to execute
+    # by rootwrap rather than use explicit whitelist of paths in
+    # rootwrap.conf
+    sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf
+
    # Set up the rootwrap sudoers
    local tempfile
    tempfile=$(mktemp)
@@ -27,6 +27,11 @@ set +o xtrace
 APACHE_USER=${APACHE_USER:-$STACK_USER}
 APACHE_GROUP=${APACHE_GROUP:-$(id -gn $APACHE_USER)}

+APACHE_LOCAL_HOST=$SERVICE_LOCAL_HOST
+if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
+    APACHE_LOCAL_HOST=[$APACHE_LOCAL_HOST]
+fi
+

 # Set up apache name and configuration directory
 # Note that APACHE_CONF_DIR is really more accurately apache's vhost
@@ -39,10 +44,6 @@ elif is_fedora; then
    APACHE_NAME=httpd
    APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/conf.d}
    APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
-elif is_suse; then
-    APACHE_NAME=apache2
-    APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/vhosts.d}
-    APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
 fi
 APACHE_LOG_DIR="/var/log/${APACHE_NAME}"

@@ -60,11 +61,6 @@ function enable_apache_mod {
            sudo a2enmod $mod
            restart_apache_server
        fi
-    elif is_suse; then
-        if ! a2enmod -q $mod ; then
-            sudo a2enmod $mod
-            restart_apache_server
-        fi
    elif is_fedora; then
        # pass
        true
@@ -82,22 +78,15 @@ function install_apache_uwsgi {
        apxs="apxs"
    fi

-    # This varies based on packaged/installed.  If we've
-    # pip_installed, then the pip setup will only build a "python"
-    # module that will be either python2 or python3 depending on what
-    # it was built with.
-    #
-    # For package installs, the distro ships both plugins and you need
-    # to select the right one ... it will not be autodetected.
-    UWSGI_PYTHON_PLUGIN=python3
-
    if is_ubuntu; then
        local pkg_list="uwsgi uwsgi-plugin-python3 libapache2-mod-proxy-uwsgi"
-        if [[ "$DISTRO" == 'bionic' ]]; then
-            pkg_list="${pkg_list} uwsgi-plugin-python"
-        fi
        install_package ${pkg_list}
-    elif is_fedora; then
+    # NOTE(ianw) 2022-02-03 : Fedora 35 needs to skip this and fall
+    # into the install-from-source because the upstream packages
+    # didn't fix Python 3.10 compatibility before release.  Should be
+    # fixed in uwsgi 4.9.0; can remove this when packages available
+    # or we drop this release
+    elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ f36 ]]; then
        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
        # default; the mod_proxy_uwsgi package actually conflicts now.
        # See:
@@ -106,10 +95,6 @@ function install_apache_uwsgi {
        # Thus there is nothing else to do after this install
        install_package uwsgi \
                        uwsgi-plugin-python3
-    elif [[ $os_VENDOR =~ openSUSE ]]; then
-        install_package uwsgi \
-                        uwsgi-python3 \
-                        apache2-mod_uwsgi
    else
        # Compile uwsgi from source.
        local dir
@@ -125,10 +110,9 @@ function install_apache_uwsgi {
        popd
        # delete the temp directory
        sudo rm -rf $dir
-        UWSGI_PYTHON_PLUGIN=python
    fi

-    if is_ubuntu || is_suse ; then
+    if is_ubuntu; then
        # we've got to enable proxy and proxy_uwsgi for this to work
        sudo a2enmod proxy
        sudo a2enmod proxy_uwsgi
@@ -153,13 +137,13 @@ function install_apache_wsgi {
    elif is_fedora; then
        sudo rm -f /etc/httpd/conf.d/000-*
        install_package httpd python3-mod_wsgi
+        # rpm distros dont enable httpd by default so enable it to support reboots.
+        sudo systemctl enable httpd
        # For consistency with Ubuntu, switch to the worker mpm, as
        # the default is event
        sudo sed -i '/mod_mpm_prefork.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
        sudo sed -i '/mod_mpm_event.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
        sudo sed -i '/mod_mpm_worker.so/s/^#//g' /etc/httpd/conf.modules.d/00-mpm.conf
-    elif is_suse; then
-        install_package apache2 apache2-mod_wsgi
    else
        exit_distro_not_supported "apache wsgi installation"
    fi
@@ -174,7 +158,7 @@ function install_apache_wsgi {
 # recognise it. a2ensite and a2dissite ignore the .conf suffix used as parameter. The default sites'
 # files are 000-default.conf and default-ssl.conf.
 #
-# On Fedora and openSUSE, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
+# On Fedora, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
 #
 # On RHEL and CentOS, things should hopefully work as in Fedora.
 #
@@ -190,7 +174,7 @@ function apache_site_config_for {
    if is_ubuntu; then
        # Ubuntu 14.04 - Apache 2.4
        echo $APACHE_CONF_DIR/${site}.conf
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        # fedora conf.d is only imported if it ends with .conf so this is approx the same
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        if [ -f $enabled_site_file ]; then
@@ -208,7 +192,7 @@ function enable_apache_site {
    enable_apache_mod version
    if is_ubuntu; then
        sudo a2ensite ${site}
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        # Do nothing if site already enabled or no site config exists
        if [[ -f ${enabled_site_file}.disabled ]] && [[ ! -f ${enabled_site_file} ]]; then
@@ -222,7 +206,7 @@ function disable_apache_site {
    local site=$@
    if is_ubuntu; then
        sudo a2dissite ${site} || true
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        # Do nothing if no site config exists
        if [[ -f ${enabled_site_file} ]]; then
@@ -253,13 +237,17 @@ function restart_apache_server {
    restart_service $APACHE_NAME
 }

+# write_uwsgi_config() - Create a new uWSGI config file
 function write_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
    local http=$4
-    local name=""
-    name=$(basename $wsgi)
+    local name=$5
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.
@@ -274,39 +262,47 @@ function write_uwsgi_config {
    local socket="$socket_dir/${name}.socket"

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
-    iniset "$file" uwsgi processes $API_WORKERS
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given. Note that the regex isn't exhaustive - neither Python modules
+    # nor Python variables can start with a number - but it's "good enough"
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
+    iniset "$conf" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
+    iniset "$conf" uwsgi lazy-apps true

    # If we said bind directly to http, then do that and don't start the apache proxy
    if [[ -n "$http" ]]; then
-        iniset "$file" uwsgi http $http
+        iniset "$conf" uwsgi http $http
    else
        local apache_conf=""
        apache_conf=$(apache_site_config_for $name)
-        iniset "$file" uwsgi socket "$socket"
-        iniset "$file" uwsgi chmod-socket 666
-        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee -a $apache_conf
+        iniset "$conf" uwsgi socket "$socket"
+        iniset "$conf" uwsgi chmod-socket 666
+        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 acquire=1 " | sudo tee -a $apache_conf
        enable_apache_site $name
        restart_apache_server
    fi
@@ -319,47 +315,58 @@ function write_uwsgi_config {
 # but that involves having apache buffer the request before sending it to
 # uwsgi.
 function write_local_uwsgi_http_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
-    name=$(basename $wsgi)
+    local name=$4
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
    port=$(get_random_port)
-    iniset "$file" uwsgi http-socket "127.0.0.1:$port"
-    iniset "$file" uwsgi processes $API_WORKERS
+    iniset "$conf" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
-    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
-    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
+    # uwsgi recommends this to prevent thundering herd on accept.
+    iniset "$conf" uwsgi thunder-lock true
+    # Set hook to trigger graceful shutdown on SIGTERM
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
-    iniset "$file" uwsgi chmod-socket 666
-    iniset "$file" uwsgi http-raw-body true
-    iniset "$file" uwsgi http-chunked-input true
-    iniset "$file" uwsgi http-auto-chunked true
-    iniset "$file" uwsgi http-keepalive false
+    iniset "$conf" uwsgi lazy-apps true
+    iniset "$conf" uwsgi chmod-socket 666
+    iniset "$conf" uwsgi http-raw-body true
+    iniset "$conf" uwsgi http-chunked-input true
+    iniset "$conf" uwsgi http-auto-chunked true
+    iniset "$conf" uwsgi http-keepalive false
    # Increase socket timeout for slow chunked uploads
-    iniset "$file" uwsgi socket-timeout 30
+    iniset "$conf" uwsgi socket-timeout 30

    enable_apache_mod proxy
    enable_apache_mod proxy_http
@@ -367,7 +374,7 @@ function write_local_uwsgi_http_config {
    apache_conf=$(apache_site_config_for $name)
    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${url}\" \"http://127.0.0.1:$port\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${url}\" \"http://$APACHE_LOCAL_HOST:$port\" retry=0 acquire=1 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }
@@ -386,18 +393,24 @@ function write_local_proxy_http_config {

    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${loc}\" \"$url\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${loc}\" \"$url\" retry=0 acquire=1 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }

 function remove_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local name=""
+    # TODO(stephenfin): Remove this call when everyone is using module path
+    # configuration instead of file path configuration
    name=$(basename $wsgi)

-    rm -rf $file
+    if [[ "$wsgi" = /* ]]; then
+        deprecated "Passing a wsgi script to remove_uwsgi_config is deprecated, pass an application name instead"
+    fi
+
+    rm -rf $conf
    disable_apache_site $name
 }

@@ -31,6 +31,7 @@ set +o xtrace
 CINDER_DRIVER=${CINDER_DRIVER:-default}
 CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins
 CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends
+CINDER_BACKUPS=$TOP_DIR/lib/cinder_backups

 # grab plugin config if specified via cinder_driver
 if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
@@ -42,6 +43,13 @@ GITDIR["python-cinderclient"]=$DEST/python-cinderclient
 GITDIR["python-brick-cinderclient-ext"]=$DEST/python-brick-cinderclient-ext
 CINDER_DIR=$DEST/cinder

+if [[ $SERVICE_IP_VERSION == 6 ]]; then
+    CINDER_MY_IP="$HOST_IPV6"
+else
+    CINDER_MY_IP="$HOST_IP"
+fi
+
+
 # Cinder virtual environment
 if [[ ${USE_VENV} = True ]]; then
    PROJECT_VENV["cinder"]=${CINDER_DIR}.venv
@@ -68,6 +76,11 @@ CINDER_SERVICE_PORT_INT=${CINDER_SERVICE_PORT_INT:-18776}
 CINDER_SERVICE_PROTOCOL=${CINDER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 CINDER_SERVICE_LISTEN_ADDRESS=${CINDER_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)}

+# We do not need to report service status every 10s for devstack-like
+# deployments. In the gate this generates extra work for the services and the
+# database which are already taxed.
+CINDER_SERVICE_REPORT_INTERVAL=${CINDER_SERVICE_REPORT_INTERVAL:-120}
+
 # What type of LVM device should Cinder use for LVM backend
 # Defaults to auto, which will do thin provisioning if it's a fresh
 # volume group, otherwise it will do thick. The other valid choices are
@@ -75,6 +88,10 @@ CINDER_SERVICE_LISTEN_ADDRESS=${CINDER_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $S
 # thin provisioning.
 CINDER_LVM_TYPE=${CINDER_LVM_TYPE:-auto}

+# ``CINDER_USE_SERVICE_TOKEN`` is a mode where service token is passed along with
+# user token while communicating to external REST APIs like Glance.
+CINDER_USE_SERVICE_TOKEN=$(trueorfalse True CINDER_USE_SERVICE_TOKEN)
+
 # Default backends
 # The backend format is type:name where type is one of the supported backend
 # types (lvm, nfs, etc) and name is the identifier used in the Cinder
@@ -87,16 +104,61 @@ CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
 CINDER_VOLUME_CLEAR=${CINDER_VOLUME_CLEAR:-${CINDER_VOLUME_CLEAR_DEFAULT:-zero}}
 CINDER_VOLUME_CLEAR=$(echo ${CINDER_VOLUME_CLEAR} | tr '[:upper:]' '[:lower:]')

-# Centos7 and OpenSUSE switched to using LIO and that's all that's supported,
-# although the tgt bits are in EPEL and OpenSUSE we don't want that for CI
-if is_fedora || is_suse; then
-    CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-lioadm}
-    if [[ ${CINDER_ISCSI_HELPER} != "lioadm" ]]; then
-        die "lioadm is the only valid Cinder target_helper config on this platform"
+VOLUME_TYPE_MULTIATTACH=${VOLUME_TYPE_MULTIATTACH:-multiattach}
+
+if [[ -n "$CINDER_ISCSI_HELPER" ]]; then
+    if [[ -z "$CINDER_TARGET_HELPER" ]]; then
+        deprecated 'Using CINDER_ISCSI_HELPER is deprecated, use CINDER_TARGET_HELPER instead'
+        CINDER_TARGET_HELPER="$CINDER_ISCSI_HELPER"
+    else
+        deprecated 'Deprecated CINDER_ISCSI_HELPER is set, but is being overwritten by CINDER_TARGET_HELPER'
    fi
-else
-    CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-tgtadm}
 fi
+CINDER_TARGET_HELPER=${CINDER_TARGET_HELPER:-lioadm}
+
+if [[ $CINDER_TARGET_HELPER == 'nvmet' ]]; then
+    CINDER_TARGET_PROTOCOL=${CINDER_TARGET_PROTOCOL:-'nvmet_rdma'}
+    CINDER_TARGET_PREFIX=${CINDER_TARGET_PREFIX:-'nvme-subsystem-1'}
+    CINDER_TARGET_PORT=${CINDER_TARGET_PORT:-4420}
+else
+    CINDER_TARGET_PROTOCOL=${CINDER_TARGET_PROTOCOL:-'iscsi'}
+    CINDER_TARGET_PREFIX=${CINDER_TARGET_PREFIX:-'iqn.2010-10.org.openstack:'}
+    CINDER_TARGET_PORT=${CINDER_TARGET_PORT:-3260}
+fi
+
+
+# EL should only use lioadm
+if is_fedora; then
+    if [[ ${CINDER_TARGET_HELPER} != "lioadm" && ${CINDER_TARGET_HELPER} != 'nvmet' ]]; then
+        die "lioadm and nvmet are the only valid Cinder target_helper config on this platform"
+    fi
+fi
+
+# When Cinder is used as a backend for Glance, it can be configured to clone
+# the volume containing image data directly in the backend instead of
+# transferring data from volume to volume.  Value is a comma separated list of
+# schemes (currently only 'file' and 'cinder' are supported).  The default
+# configuration in Cinder is empty (that is, do not use this feature).  NOTE:
+# to use this feature you must also enable GLANCE_SHOW_DIRECT_URL and/or
+# GLANCE_SHOW_MULTIPLE_LOCATIONS for glance-api.conf.
+CINDER_ALLOWED_DIRECT_URL_SCHEMES=${CINDER_ALLOWED_DIRECT_URL_SCHEMES:-}
+if [[ -n "$CINDER_ALLOWED_DIRECT_URL_SCHEMES" ]]; then
+    if [[ "${GLANCE_SHOW_DIRECT_URL:-False}" != "True" \
+            && "${GLANCE_SHOW_MULTIPLE_LOCATIONS:-False}" != "True" ]]; then
+        warn $LINENO "CINDER_ALLOWED_DIRECT_URL_SCHEMES is set, but neither \
+GLANCE_SHOW_DIRECT_URL nor GLANCE_SHOW_MULTIPLE_LOCATIONS is True"
+    fi
+fi
+
+# For backward compatibility
+# Before CINDER_BACKUP_DRIVER was introduced, ceph backup driver was configured
+# along with ceph backend driver.
+if [[ -z "${CINDER_BACKUP_DRIVER}" && "$CINDER_ENABLED_BACKENDS" =~ "ceph" ]]; then
+    CINDER_BACKUP_DRIVER=ceph
+fi
+
+# Supported backup drivers are in lib/cinder_backups
+CINDER_BACKUP_DRIVER=${CINDER_BACKUP_DRIVER:-swift}

 # Toggle for deploying Cinder under a wsgi server. Legacy mod_wsgi
 # reference should be cleaned up to more accurately refer to uwsgi.
@@ -113,9 +175,24 @@ if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
    done
 fi

+# Source the backup driver
+if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+    if [[ -r $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER ]]; then
+        source $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER
+    else
+        die "cinder backup driver $CINDER_BACKUP_DRIVER is not supported"
+    fi
+fi
+
 # Environment variables to configure the image-volume cache
 CINDER_IMG_CACHE_ENABLED=${CINDER_IMG_CACHE_ENABLED:-True}

+# Environment variables to configure the optimized volume upload
+CINDER_UPLOAD_OPTIMIZED=${CINDER_UPLOAD_OPTIMIZED:-False}
+
+# Environment variables to configure the internal tenant during optimized volume upload
+CINDER_UPLOAD_INTERNAL_TENANT=${CINDER_UPLOAD_INTERNAL_TENANT:-False}
+
 # For limits, if left unset, it will use cinder defaults of 0 for unlimited
 CINDER_IMG_CACHE_SIZE_GB=${CINDER_IMG_CACHE_SIZE_GB:-}
 CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
@@ -125,6 +202,17 @@ CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
 # enable the cache for all cinder backends.
 CINDER_CACHE_ENABLED_FOR_BACKENDS=${CINDER_CACHE_ENABLED_FOR_BACKENDS:-$CINDER_ENABLED_BACKENDS}

+# Configure which cinder backends will have optimized volume upload, this takes the same
+# form as the CINDER_ENABLED_BACKENDS config option. By default it will
+# enable the cache for all cinder backends.
+CINDER_UPLOAD_OPTIMIZED_BACKENDS=${CINDER_UPLOAD_OPTIMIZED_BACKENDS:-$CINDER_ENABLED_BACKENDS}
+
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the  Volume API policies to start checking the scope of token. by default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+CINDER_ENFORCE_SCOPE=$(trueorfalse False CINDER_ENFORCE_SCOPE)
+
 # Functions
 # ---------

@@ -146,7 +234,7 @@ function _cinder_cleanup_apache_wsgi {
 function cleanup_cinder {
    # ensure the volume group is cleared up because fails might
    # leave dead volumes in the group
-    if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
+    if [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
        local targets
        targets=$(sudo tgtadm --op show --mode target)
        if [ $? -ne 0 ]; then
@@ -174,8 +262,14 @@ function cleanup_cinder {
        else
            stop_service tgtd
        fi
-    else
+    elif [ "$CINDER_TARGET_HELPER" = "lioadm" ]; then
        sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
+    elif [ "$CINDER_TARGET_HELPER" = "nvmet" ]; then
+        # If we don't disconnect everything vgremove will block
+        sudo nvme disconnect-all
+        sudo nvmetcli clear
+    else
+        die $LINENO "Unknown value \"$CINDER_TARGET_HELPER\" for CINDER_TARGET_HELPER"
    fi

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
@@ -189,8 +283,14 @@ function cleanup_cinder {
        done
    fi

+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type cleanup_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            cleanup_cinder_backup_$CINDER_BACKUP_DRIVER
+        fi
+    fi
+
    stop_process "c-api"
-    remove_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI"
+    remove_uwsgi_config "$CINDER_UWSGI_CONF" "cinder-wsgi"
 }

 # configure_cinder() - Set config files, create data dirs, etc
@@ -220,7 +320,7 @@ function configure_cinder {

    iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

-    iniset $CINDER_CONF DEFAULT target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF DEFAULT target_helper "$CINDER_TARGET_HELPER"
    iniset $CINDER_CONF database connection `database_connection_url cinder`
    iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
    iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"
@@ -228,13 +328,25 @@ function configure_cinder {
    iniset $CINDER_CONF DEFAULT osapi_volume_listen $CINDER_SERVICE_LISTEN_ADDRESS
    iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH
    iniset $CINDER_CONF oslo_concurrency lock_path $CINDER_STATE_PATH
-    if [[ $SERVICE_IP_VERSION == 6 ]]; then
-        iniset $CINDER_CONF DEFAULT my_ip "$HOST_IPV6"
-    else
-        iniset $CINDER_CONF DEFAULT my_ip "$HOST_IP"
-    fi
+    iniset $CINDER_CONF DEFAULT my_ip "$CINDER_MY_IP"
    iniset $CINDER_CONF key_manager backend cinder.keymgr.conf_key_mgr.ConfKeyManager
    iniset $CINDER_CONF key_manager fixed_key $(openssl rand -hex 16)
+    if [[ -n "$CINDER_ALLOWED_DIRECT_URL_SCHEMES" ]]; then
+        iniset $CINDER_CONF DEFAULT allowed_direct_url_schemes $CINDER_ALLOWED_DIRECT_URL_SCHEMES
+    fi
+
+    # set default quotas
+    iniset $CINDER_CONF DEFAULT quota_volumes ${CINDER_QUOTA_VOLUMES:-10}
+    iniset $CINDER_CONF DEFAULT quota_backups ${CINDER_QUOTA_BACKUPS:-10}
+    iniset $CINDER_CONF DEFAULT quota_snapshots ${CINDER_QUOTA_SNAPSHOTS:-10}
+
+    # Avoid RPC timeouts in slow CI and test environments by doubling the
+    # default response timeout set by RPC clients. See bug #1873234 for more
+    # details and example failures.
+    iniset $CINDER_CONF DEFAULT rpc_response_timeout 120
+
+    iniset $CINDER_CONF DEFAULT report_interval $CINDER_SERVICE_REPORT_INTERVAL
+    iniset $CINDER_CONF DEFAULT service_down_time $(($CINDER_SERVICE_REPORT_INTERVAL * 6))

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local enabled_backends=""
@@ -250,24 +362,28 @@ function configure_cinder {
                default_name=$be_name
            fi
            enabled_backends+=$be_name,
-
-            iniset $CINDER_CONF $be_name volume_clear $CINDER_VOLUME_CLEAR
-
        done
        iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}
        if [[ -n "$default_name" ]]; then
            iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}
        fi
        configure_cinder_image_volume_cache
+
+        # The upload optimization uses Cinder's clone volume functionality to
+        # clone the Image-Volume from source volume hence can only be
+        # performed when glance is using cinder as it's backend.
+        if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then
+            # Configure optimized volume upload
+            configure_cinder_volume_upload
+        fi
    fi

-    if is_service_enabled c-bak; then
-        # NOTE(mriedem): The default backup driver uses swift and if we're
-        # on a subnode we might not know if swift is enabled, but chances are
-        # good that it is on the controller so configure the backup service
-        # to use it. If we want to configure the backup service to use
-        # a non-swift driver, we'll likely need environment variables.
-        iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type configure_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            configure_cinder_backup_$CINDER_BACKUP_DRIVER
+        else
+            die "configure_cinder_backup_$CINDER_BACKUP_DRIVER doesn't exist in $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER"
+        fi
    fi

    if is_service_enabled ceilometer; then
@@ -297,7 +413,9 @@ function configure_cinder {
    # Format logging
    setup_logging $CINDER_CONF $CINDER_USE_MOD_WSGI

-    write_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI" "/volume"
+    if is_service_enabled c-api; then
+        write_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI" "/volume"
+    fi

    if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
        configure_cinder_driver
@@ -319,61 +437,58 @@ function configure_cinder {
    if [[ ! -z "$CINDER_COORDINATION_URL" ]]; then
        iniset $CINDER_CONF coordination backend_url "$CINDER_COORDINATION_URL"
    elif is_service_enabled etcd3; then
-        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT"
+        # NOTE(jan.gutter): api_version can revert to default once tooz is
+        # updated with the etcd v3.4 defaults
+        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT?api_version=v3"
+    fi
+
+    if [[ "$CINDER_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $CINDER_CONF oslo_policy enforce_scope true
+        iniset $CINDER_CONF oslo_policy enforce_new_defaults true
+    else
+        iniset $CINDER_CONF oslo_policy enforce_scope false
+        iniset $CINDER_CONF oslo_policy enforce_new_defaults false
+    fi
+
+    if [ "$CINDER_USE_SERVICE_TOKEN" == "True" ]; then
+        init_cinder_service_user_conf
    fi
 }

 # create_cinder_accounts() - Set up common required cinder accounts

-# Tenant               User       Roles
+# Project              User       Roles
 # ------------------------------------------------------------------
-# service              cinder     admin        # if enabled
+# SERVICE_PROJECT_NAME cinder     service
+# SERVICE_PROJECT_NAME cinder     creator (if Barbican is enabled)

 # Migrated from keystone_data.sh
 function create_cinder_accounts {
    # Cinder
    if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

-        create_service_user "cinder"
+        local extra_role=""
+
+        # cinder needs the "creator" role in order to interact with barbican
+        if is_service_enabled barbican; then
+            extra_role=$(get_or_create_role "creator")
+        fi
+
+        create_service_user "cinder" $extra_role
+
+        local cinder_api_url
+        if [[ "$CINDER_USE_MOD_WSGI" == "False" ]]; then
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT"
+        else
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume"
+        fi

        # block-storage is the official service type
        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
-        if [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv2" "volumev2" "Cinder Volume Service V2"
-            get_or_create_endpoint \
-                "volumev2" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-        else
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv2" "volumev2" "Cinder Volume Service V2"
-            get_or_create_endpoint \
-                "volumev2" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v2/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-        fi
-
+        get_or_create_endpoint \
+            "block-storage" \
+            "$REGION_NAME" \
+            "$cinder_api_url/v3"
        configure_cinder_internal_tenant
    fi
 }
@@ -396,15 +511,17 @@ function init_cinder {
            be_type=${be%%:*}
            be_name=${be##*:}
            if type init_cinder_backend_${be_type} >/dev/null 2>&1; then
-                # Always init the default volume group for lvm.
-                if [[ "$be_type" == "lvm" ]]; then
-                    init_default_lvm_volume_group
-                fi
                init_cinder_backend_${be_type} ${be_name}
            fi
        done
    fi

+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type init_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            init_cinder_backup_$CINDER_BACKUP_DRIVER
+        fi
+    fi
+
    mkdir -p $CINDER_STATE_PATH/volumes
 }

@@ -412,9 +529,9 @@ function init_cinder {
 function install_cinder {
    git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH
    setup_develop $CINDER_DIR
-    if [[ "$CINDER_ISCSI_HELPER" == "tgtadm" ]]; then
+    if [[ "$CINDER_TARGET_HELPER" == "tgtadm" ]]; then
        install_package tgt
-    elif [[ "$CINDER_ISCSI_HELPER" == "lioadm" ]]; then
+    elif [[ "$CINDER_TARGET_HELPER" == "lioadm" ]]; then
        if is_ubuntu; then
            # TODO(frickler): Workaround for https://launchpad.net/bugs/1819819
            sudo mkdir -p /etc/target
@@ -423,6 +540,43 @@ function install_cinder {
        else
            install_package targetcli
        fi
+    elif [[ "$CINDER_TARGET_HELPER" == "nvmet" ]]; then
+        install_package nvme-cli
+
+        # TODO: Remove manual installation of the dependency when the
+        # requirement is added to nvmetcli:
+        # http://lists.infradead.org/pipermail/linux-nvme/2022-July/033576.html
+        if is_ubuntu; then
+            install_package python3-configshell-fb
+        else
+            install_package python3-configshell
+        fi
+        # Install from source because Ubuntu doesn't have the package and some packaged versions didn't work on Python 3
+        pip_install git+git://git.infradead.org/users/hch/nvmetcli.git
+
+        sudo modprobe nvmet
+        sudo modprobe nvme-fabrics
+
+        if [[ $CINDER_TARGET_PROTOCOL == 'nvmet_rdma' ]]; then
+            install_package rdma-core
+            sudo modprobe nvme-rdma
+
+            # Create the Soft-RoCE device over the networking interface
+            local iface=${HOST_IP_IFACE:-`ip -br -$SERVICE_IP_VERSION a | grep $CINDER_MY_IP | awk '{print $1}'`}
+            if [[ -z "$iface" ]]; then
+                die $LINENO "Cannot find interface to bind Soft-RoCE"
+            fi
+
+            if ! sudo rdma link | grep $iface ; then
+                sudo rdma link add rxe_$iface type rxe netdev $iface
+            fi
+
+        elif [[ $CINDER_TARGET_PROTOCOL == 'nvmet_tcp' ]]; then
+            sudo modprobe nvme-tcp
+
+        else  # 'nvmet_fc'
+            sudo modprobe nvme-fc
+        fi
    fi
 }

@@ -459,18 +613,13 @@ function start_cinder {
        service_port=$CINDER_SERVICE_PORT_INT
        service_protocol="http"
    fi
-    if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
+    if [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
        if is_service_enabled c-vol; then
            # Delete any old stack.conf
            sudo rm -f /etc/tgt/conf.d/stack.conf
            _configure_tgt_for_config_d
            if is_ubuntu; then
                sudo service tgt restart
-            elif is_suse; then
-                # NOTE(dmllr): workaround restart bug
-                # https://bugzilla.suse.com/show_bug.cgi?id=934642
-                stop_service tgtd
-                start_service tgtd
            else
                restart_service tgtd
            fi
@@ -499,8 +648,13 @@ function start_cinder {
    fi

    run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
-    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
-    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
+    # Tune glibc for Python Services using single malloc arena for all threads
+    # and disabling dynamic thresholds to reduce memory usage when using native
+    # threads directly or via eventlet.tpool
+    # https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html
+    malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144"
+    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning"
+    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning"

    # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
    # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
@@ -515,6 +669,23 @@ function stop_cinder {
    stop_process c-vol
 }

+function create_one_type {
+    type_name=$1
+    property_key=$2
+    property_value=$3
+    # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
+    if is_service_enabled keystone; then
+        openstack --os-region-name="$REGION_NAME" volume type create --property $property_key="$property_value" $type_name
+    else
+        # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
+        # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
+        local cinder_url
+        cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create $type_name
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key $type_name set $property_key="$property_value"
+    fi
+}
+
 # create_volume_types() - Create Cinder's configured volume types
 function create_volume_types {
    # Create volume types
@@ -522,18 +693,20 @@ function create_volume_types {
        local be be_name
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_name=${be##*:}
-            # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
-            if is_service_enabled keystone; then
-                openstack --os-region-name="$REGION_NAME" volume type create --property volume_backend_name="${be_name}" ${be_name}
-            else
-                # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
-                # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
-                local cinder_url
-                cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create ${be_name}
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key ${be_name} set volume_backend_name=${be_name}
-            fi
+            create_one_type $be_name "volume_backend_name" $be_name
        done
+
+        if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then
+            create_one_type $VOLUME_TYPE_MULTIATTACH $VOLUME_TYPE_MULTIATTACH "<is> True"
+        fi
+
+        # Increase quota for the service project if glance is using cinder,
+        # since it's likely to occasionally go above the default 10 in parallel
+        # test execution.
+        if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then
+            openstack --os-region-name="$REGION_NAME" \
+                      quota set --volumes 50 "$SERVICE_PROJECT_NAME"
+        fi
    fi
 }

@@ -570,6 +743,24 @@ function configure_cinder_image_volume_cache {
    done
 }

+function configure_cinder_volume_upload {
+    # Expect UPLOAD_VOLUME_OPTIMIZED_FOR_BACKENDS to be a list of backends
+    # similar to CINDER_ENABLED_BACKENDS with NAME:TYPE where NAME will
+    # be the backend specific configuration stanza in cinder.conf.
+    local be be_name
+    for be in ${CINDER_UPLOAD_OPTIMIZED_BACKENDS//,/ }; do
+        be_name=${be##*:}
+
+        iniset $CINDER_CONF $be_name image_upload_use_cinder_backend $CINDER_UPLOAD_OPTIMIZED
+        iniset $CINDER_CONF $be_name image_upload_use_internal_tenant $CINDER_UPLOAD_INTERNAL_TENANT
+    done
+}
+
+function init_cinder_service_user_conf {
+    configure_keystone_authtoken_middleware $CINDER_CONF cinder service_user
+    iniset $CINDER_CONF service_user send_service_user_token True
+    iniset $CINDER_CONF service_user auth_strategy keystone
+}

 # Restore xtrace
 $_XTRACE_CINDER
@@ -6,12 +6,6 @@
 # Enable with:
 #
 #   CINDER_ENABLED_BACKENDS+=,ceph:ceph
-#
-# Optional parameters:
-#   CINDER_BAK_CEPH_POOL=<pool-name>
-#   CINDER_BAK_CEPH_USER=<user>
-#   CINDER_BAK_CEPH_POOL_PG=<pg-num>
-#   CINDER_BAK_CEPH_POOL_PGP=<pgp-num>

 # Dependencies:
 #
@@ -29,11 +23,6 @@ set +o xtrace
 # Defaults
 # --------

-CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups}
-CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8}
-CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8}
-CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}
-

 # Entry Points
 # ------------
@@ -52,27 +41,6 @@ function configure_cinder_backend_ceph {
    iniset $CINDER_CONF $be_name rbd_flatten_volume_from_snapshot False
    iniset $CINDER_CONF $be_name rbd_max_clone_depth 5
    iniset $CINDER_CONF DEFAULT glance_api_version 2
-
-    if is_service_enabled c-bak; then
-        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
-        if [ "$REMOTE_CEPH" = "False" ]; then
-            # Configure Cinder backup service options, ceph pool, ceph user and ceph key
-            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} size ${CEPH_REPLICAS}
-            if [[ $CEPH_REPLICAS -ne 1 ]]; then
-                sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
-            fi
-        fi
-        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-
-        iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
-        iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
-        iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL"
-        iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER"
-        iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0
-        iniset $CINDER_CONF DEFAULT backup_ceph_stripe_count 0
-        iniset $CINDER_CONF DEFAULT restore_discard_excess_bytes True
-    fi
 }

 # Restore xtrace
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# lib/cinder_backends/ceph_iscsi
+# Configure the ceph_iscsi backend
+
+# Enable with:
+#
+#   CINDER_ENABLED_BACKENDS+=,ceph_iscsi:ceph_iscsi
+#
+# Optional paramteters:
+#   CEPH_ISCSI_API_URL=<url to the rbd-target-api service>
+#
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# configure_ceph_backend_ceph_iscsi - called from configure_cinder()
+
+
+# Save trace setting
+_XTRACE_CINDER_CEPH_ISCSI=$(set +o | grep xtrace)
+set +o xtrace
+
+# Entry Points
+# ------------
+
+# configure_cinder_backend_ceph_iscsi - Set config files, create data dirs, etc
+# configure_cinder_backend_ceph_iscsi $name
+function configure_cinder_backend_ceph_iscsi {
+    local be_name=$1
+
+    CEPH_ISCSI_API_URL=${CEPH_ISCSI_API_URL:-http://$CEPH_ISCSI_API_HOST:$CEPH_ISCSI_API_PORT}
+
+    iniset $CINDER_CONF $be_name volume_backend_name $be_name
+    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.ceph.rbd_iscsi.RBDISCSIDriver"
+    iniset $CINDER_CONF $be_name rbd_ceph_conf "$CEPH_CONF_FILE"
+    iniset $CINDER_CONF $be_name rbd_pool "$CINDER_CEPH_POOL"
+    iniset $CINDER_CONF $be_name rbd_user "$CINDER_CEPH_USER"
+    iniset $CINDER_CONF $be_name rbd_iscsi_api_user "$CEPH_ISCSI_API_USER"
+    iniset $CINDER_CONF $be_name rbd_iscsi_api_password "$CEPH_ISCSI_API_PASSWORD"
+    iniset $CINDER_CONF $be_name rbd_iscsi_api_url "$CEPH_ISCSI_API_URL"
+    iniset $CINDER_CONF $be_name rbd_iscsi_target_iqn "$CEPH_ISCSI_TARGET_IQN"
+    iniset $CINDER_CONF $be_name rbd_flatten_volume_from_snapshot False
+    iniset $CINDER_CONF $be_name rbd_max_clone_depth 5
+    iniset $CINDER_CONF DEFAULT glance_api_version 2
+
+    pip_install rbd-iscsi-client
+}
+
+# Restore xtrace
+$_XTRACE_CINDER_CEPH_ISCSI
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -50,7 +50,7 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.tests.fake_driver.FakeGateDriver"
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
-    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF $be_name target_helper "$CINDER_TARGET_HELPER"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"

    if [[ "$CINDER_VOLUME_CLEAR" == "non" ]]; then
@@ -50,9 +50,12 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMVolumeDriver"
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
-    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF $be_name target_helper "$CINDER_TARGET_HELPER"
+    iniset $CINDER_CONF $be_name target_protocol "$CINDER_TARGET_PROTOCOL"
+    iniset $CINDER_CONF $be_name target_port "$CINDER_TARGET_PORT"
+    iniset $CINDER_CONF $be_name target_prefix "$CINDER_TARGET_PREFIX"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"
-
+    iniset $CINDER_CONF $be_name volume_clear "$CINDER_VOLUME_CLEAR"
 }

 # init_cinder_backend_lvm - Initialize volume group
@@ -32,6 +32,15 @@ function configure_cinder_backend_nfs {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.nfs.NfsDriver"
    iniset $CINDER_CONF $be_name nfs_shares_config "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
+    iniset $CINDER_CONF $be_name nas_host localhost
+    iniset $CINDER_CONF $be_name nas_share_path ${NFS_EXPORT_DIR}
+    iniset $CINDER_CONF $be_name nas_secure_file_operations \
+        ${NFS_SECURE_FILE_OPERATIONS}
+    iniset $CINDER_CONF $be_name nas_secure_file_permissions \
+        ${NFS_SECURE_FILE_PERMISSIONS}
+
+    # NFS snapshot support is currently opt-in only.
+    iniset $CINDER_CONF $be_name nfs_snapshot_support True

    echo "$CINDER_NFS_SERVERPATH" | tee "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
 }
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# lib/cinder_backups/ceph
+# Configure the ceph backup driver
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=ceph
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_CEPH=$(set +o | grep xtrace)
+set +o xtrace
+
+# Defaults
+# --------
+
+CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups}
+CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8}
+CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8}
+CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}
+
+
+function configure_cinder_backup_ceph {
+    # Execute this part only when cephadm is not used
+    if [[ "$CEPHADM_DEPLOY" = "False" ]]; then
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
+        if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
+            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+        fi
+        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+    fi
+
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
+    iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL"
+    iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER"
+    iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0
+    iniset $CINDER_CONF DEFAULT backup_ceph_stripe_count 0
+    iniset $CINDER_CONF DEFAULT restore_discard_excess_bytes True
+}
+
+# init_cinder_backup_ceph: nothing to do
+# cleanup_cinder_backup_ceph: nothing to do
+
+# Restore xtrace
+$_XTRACE_CINDER_CEPH
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# lib/cinder_backups/s3_swift
+# Configure the s3 backup driver with swift s3api
+#
+# TODO: create lib/cinder_backup/s3 for external s3 compatible storage
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=s3_swift
+#   enable_service s3api s-proxy s-object s-container s-account
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_S3_SWIFT=$(set +o | grep xtrace)
+set +o xtrace
+
+function configure_cinder_backup_s3_swift {
+    # This configuration requires swift and s3api. If we're
+    # on a subnode we might not know if they are enabled
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.s3.S3BackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_s3_endpoint_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$S3_SERVICE_PORT"
+}
+
+function init_cinder_backup_s3_swift {
+    openstack ec2 credential create
+    iniset $CINDER_CONF DEFAULT backup_s3_store_access_key "$(openstack ec2 credential list -c Access -f value)"
+    iniset $CINDER_CONF DEFAULT backup_s3_store_secret_key "$(openstack ec2 credential list -c Secret -f value)"
+    if is_service_enabled tls-proxy; then
+        iniset $CINDER_CONF DEFAULT backup_s3_ca_cert_file "$SSL_BUNDLE_FILE"
+    fi
+}
+
+# cleanup_cinder_backup_s3_swift: nothing to do
+
+# Restore xtrace
+$_XTRACE_CINDER_S3_SWIFT
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# lib/cinder_backups/swift
+# Configure the swift backup driver
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=swift
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_SWIFT=$(set +o | grep xtrace)
+set +o xtrace
+
+
+function configure_cinder_backup_swift {
+    # NOTE(mriedem): The default backup driver uses swift and if we're
+    # on a subnode we might not know if swift is enabled, but chances are
+    # good that it is on the controller so configure the backup service
+    # to use it.
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.swift.SwiftBackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    if is_service_enabled tls-proxy; then
+        iniset $CINDER_CONF DEFAULT backup_swift_ca_cert_file $SSL_BUNDLE_FILE
+    fi
+}
+
+# init_cinder_backup_swift: nothing to do
+# cleanup_cinder_backup_swift: nothing to do
+
+
+# Restore xtrace
+$_XTRACE_CINDER_SWIFT
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -1,46 +0,0 @@
-#!/bin/bash
-#
-# lib/cinder_plugins/XenAPINFS
-# Configure the XenAPINFS driver
-
-# Enable with:
-#
-#   CINDER_DRIVER=XenAPINFS
-
-# Dependencies:
-#
-# - ``functions`` file
-# - ``cinder`` configurations
-
-# configure_cinder_driver - make configuration changes, including those to other services
-
-# Save trace setting
-_XTRACE_CINDER_XENAPINFS=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-# Set up default directories
-
-
-# Entry Points
-# ------------
-
-# configure_cinder_driver - Set config files, create data dirs, etc
-function configure_cinder_driver {
-    iniset $CINDER_CONF DEFAULT volume_driver "cinder.volume.drivers.xenapi.sm.XenAPINFSDriver"
-    iniset $CINDER_CONF DEFAULT xenapi_connection_url "$CINDER_XENAPI_CONNECTION_URL"
-    iniset $CINDER_CONF DEFAULT xenapi_connection_username "$CINDER_XENAPI_CONNECTION_USERNAME"
-    iniset $CINDER_CONF DEFAULT xenapi_connection_password "$CINDER_XENAPI_CONNECTION_PASSWORD"
-    iniset $CINDER_CONF DEFAULT xenapi_nfs_server "$CINDER_XENAPI_NFS_SERVER"
-    iniset $CINDER_CONF DEFAULT xenapi_nfs_serverpath "$CINDER_XENAPI_NFS_SERVERPATH"
-}
-
-# Restore xtrace
-$_XTRACE_CINDER_XENAPINFS
-
-# Local variables:
-# mode: shell-script
-# End:
@@ -89,6 +89,10 @@ function initialize_database_backends {
        DATABASE_PASSWORD=$MYSQL_PASSWORD
    fi

+    return 0
+}
+
+function define_database_baseurl {
    # We configure Nova, Horizon, Glance and Keystone to use MySQL as their
    # database server.  While they share a single server, each has their own
    # database and tables.
@@ -100,8 +104,6 @@ function initialize_database_backends {

    # NOTE: Don't specify ``/db`` in this string so we can use it for multiple services
    BASE_SQL_CONN=${BASE_SQL_CONN:-$(get_database_type_$DATABASE_TYPE)://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST}
-
-    return 0
 }

 # Recreate a given database
@@ -12,6 +12,7 @@ _XTRACE_DB_MYSQL=$(set +o | grep xtrace)
 set +o xtrace

 MYSQL_DRIVER=${MYSQL_DRIVER:-PyMySQL}
+INSTALL_DATABASE_SERVER_PACKAGES=$(trueorfalse True INSTALL_DATABASE_SERVER_PACKAGES)

 register_database mysql

@@ -19,11 +20,7 @@ if [[ -z "$MYSQL_SERVICE_NAME" ]]; then
    MYSQL_SERVICE_NAME=mysql
    if is_fedora && ! is_oraclelinux; then
        MYSQL_SERVICE_NAME=mariadb
-    elif is_suse && systemctl list-unit-files | grep -q 'mariadb\.service'; then
-        # Older mariadb packages on SLES 12 provided mysql.service.  The
-        # newer ones on SLES 12 and 15 use mariadb.service; they also
-        # provide a mysql.service symlink for backwards-compatibility, but
-        # let's not rely on that.
+    elif [[ "$DISTRO" =~ bookworm|bullseye ]]; then
        MYSQL_SERVICE_NAME=mariadb
    fi
 fi
@@ -51,7 +48,7 @@ function cleanup_database_mysql {
    elif is_oraclelinux; then
        uninstall_package mysql-community-server
        sudo rm -rf /var/lib/mysql
-    elif is_suse || is_fedora; then
+    elif is_fedora; then
        uninstall_package mariadb-server
        sudo rm -rf /var/lib/mysql
    else
@@ -66,12 +63,12 @@ function recreate_database_mysql {
 }

 function configure_database_mysql {
-    local my_conf mysql slow_log
+    local my_conf mysql slow_log my_client_conf
    echo_summary "Configuring and starting MySQL"

    if is_ubuntu; then
        my_conf=/etc/mysql/my.cnf
-    elif is_suse || is_oraclelinux; then
+    elif is_oraclelinux; then
        my_conf=/etc/my.cnf
    elif is_fedora; then
        my_conf=/etc/my.cnf
@@ -83,42 +80,69 @@ function configure_database_mysql {
        exit_distro_not_supported "mysql configuration"
    fi

-    # Start mysql-server
-    if is_fedora || is_suse; then
+    # Set fips mode on
+    if is_ubuntu; then
+        if is_fips_enabled; then
+            my_client_conf=/etc/mysql/mysql.conf.d/mysql.cnf
+            iniset -sudo $my_client_conf mysql ssl-fips-mode "on"
+            iniset -sudo $my_conf mysqld ssl-fips-mode "on"
+        fi
+    fi
+
+    # Change bind-address from localhost (127.0.0.1) to any (::)
+    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"
+
+    # (Re)Start mysql-server
+    if is_fedora; then
        # service is not started by default
        start_service $MYSQL_SERVICE_NAME
+    elif is_ubuntu; then
+        # required since bind-address could have changed above
+        restart_service $MYSQL_SERVICE_NAME
    fi

    # Set the root password - only works the first time. For Ubuntu, we already
    # did that with debconf before installing the package, but we still try,
-    # because the package might have been installed already.
-    sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+    # because the package might have been installed already. We don't do this
+    # for Ubuntu 22.04 (jammy) because the authorization model change in
+    # version 10.4 of mariadb. See
+    # https://mariadb.org/authentication-in-mariadb-10-4/
+    if ! (is_ubuntu && [[ "$DISTRO" == "jammy" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then
+        sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+    fi

    # In case of Mariadb, giving hostname in arguments causes permission
    # problems as it expects connection through socket
    if is_ubuntu && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
        local cmd_args="-uroot -p$DATABASE_PASSWORD "
    else
-        local cmd_args="-uroot -p$DATABASE_PASSWORD -h127.0.0.1 "
+        local cmd_args="-uroot -p$DATABASE_PASSWORD -h$SERVICE_LOCAL_HOST "
    fi

    # In mariadb e.g. on Ubuntu socket plugin is used for authentication
    # as root so it works only as sudo. To restore old "mysql like" behaviour,
    # we need to change auth plugin for root user
-    if is_ubuntu && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
-        sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
-        sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+    # TODO(frickler): simplify this logic
+    if is_ubuntu && [[ ! "$DISTRO" =~ bookworm|bullseye ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
+        if [[ "$DISTRO" == "jammy" ]]; then
+            # For Ubuntu 22.04 (jammy) we follow the model outlined in
+            # https://mariadb.org/authentication-in-mariadb-10-4/
+            sudo mysql -e "ALTER USER $DATABASE_USER@localhost IDENTIFIED VIA mysql_native_password USING PASSWORD('$DATABASE_PASSWORD');"
+        else
+            sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
+            sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+        fi
+    fi
+    if ! (is_ubuntu && [[ "$DISTRO" == "jammy" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then
+        # Create DB user if it does not already exist
+        sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
+        # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
+        sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"
    fi
-    # Create DB user if it does not already exist
-    sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
-    # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
-    sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"

    # Now update ``my.cnf`` for some local needs and restart the mysql service

-    # Change bind-address from localhost (127.0.0.1) to any (::) and
-    # set default db type to InnoDB
-    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"
+    # Set default db type to InnoDB
    iniset -sudo $my_conf mysqld sql_mode TRADITIONAL
    iniset -sudo $my_conf mysqld default-storage-engine InnoDB
    iniset -sudo $my_conf mysqld max_connections 1024
@@ -143,6 +167,29 @@ function configure_database_mysql {
        iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1
    fi

+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        echo "enabling MySQL performance counting"
+
+        # Install our sqlalchemy plugin
+        pip_install ${TOP_DIR}/tools/dbcounter
+
+        # Create our stats database for accounting
+        recreate_database stats
+        mysql -u $DATABASE_USER -p$DATABASE_PASSWORD -h $MYSQL_HOST -e \
+              "CREATE TABLE queries (db VARCHAR(32), op VARCHAR(32),
+                count INT, PRIMARY KEY (db, op)) ENGINE MEMORY" stats
+    fi
+
+    if [[ "$MYSQL_REDUCE_MEMORY" == "True" ]]; then
+        iniset -sudo $my_conf mysqld read_buffer_size 64K
+        iniset -sudo $my_conf mysqld innodb_buffer_pool_size 16M
+        iniset -sudo $my_conf mysqld thread_stack 192K
+        iniset -sudo $my_conf mysqld thread_cache_size 8
+        iniset -sudo $my_conf mysqld tmp_table_size 8M
+        iniset -sudo $my_conf mysqld sort_buffer_size 8M
+        iniset -sudo $my_conf mysqld max_allowed_packet 8M
+    fi
+
    restart_service $MYSQL_SERVICE_NAME
 }

@@ -173,18 +220,17 @@ EOF
        chmod 0600 $HOME/.my.cnf
    fi
    # Install mysql-server
-    if is_oraclelinux; then
-        install_package mysql-community-server
-    elif is_fedora; then
-        install_package mariadb-server mariadb-devel
-        sudo systemctl enable $MYSQL_SERVICE_NAME
-    elif is_suse; then
-        install_package mariadb-server
-        sudo systemctl enable $MYSQL_SERVICE_NAME
-    elif is_ubuntu; then
-        install_package $MYSQL_SERVICE_NAME-server
-    else
-        exit_distro_not_supported "mysql installation"
+    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
+        if is_oraclelinux; then
+            install_package mysql-community-server
+        elif is_fedora; then
+            install_package mariadb-server mariadb-devel mariadb
+            sudo systemctl enable $MYSQL_SERVICE_NAME
+        elif is_ubuntu; then
+            install_package $MYSQL_SERVICE_NAME-server
+        else
+            exit_distro_not_supported "mysql installation"
+        fi
    fi
 }

@@ -200,7 +246,17 @@ function install_database_python_mysql {

 function database_connection_url_mysql {
    local db=$1
-    echo "$BASE_SQL_CONN/$db?charset=utf8"
+    local plugin
+
+    # NOTE(danms): We don't enable perf on subnodes yet because the
+    # plugin is not installed there
+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        if is_service_enabled mysql; then
+            plugin="&plugin=dbcounter"
+        fi
+    fi
+
+    echo "$BASE_SQL_CONN/$db?charset=utf8$plugin"
 }


@@ -13,7 +13,7 @@ set +o xtrace


 MAX_DB_CONNECTIONS=${MAX_DB_CONNECTIONS:-200}
-
+INSTALL_DATABASE_SERVER_PACKAGES=$(trueorfalse True INSTALL_DATABASE_SERVER_PACKAGES)

 register_database postgresql

@@ -32,7 +32,7 @@ function cleanup_database_postgresql {
        # Get ruthless with mysql
        apt_get purge -y postgresql*
        return
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        uninstall_package postgresql-server
    else
        return
@@ -46,6 +46,10 @@ function recreate_database_postgresql {
    createdb -h $DATABASE_HOST -U$DATABASE_USER -l C -T template0 -E utf8 $db
 }

+function _exit_pg_init {
+    sudo cat /var/lib/pgsql/initdb_postgresql.log
+}
+
 function configure_database_postgresql {
    local pg_conf pg_dir pg_hba check_role version
    echo_summary "Configuring and starting PostgreSQL"
@@ -53,7 +57,9 @@ function configure_database_postgresql {
        pg_hba=/var/lib/pgsql/data/pg_hba.conf
        pg_conf=/var/lib/pgsql/data/postgresql.conf
        if ! sudo [ -e $pg_hba ]; then
+            trap _exit_pg_init EXIT
            sudo postgresql-setup initdb
+            trap - EXIT
        fi
    elif is_ubuntu; then
        version=`psql --version | cut -d ' ' -f3 | cut -d. -f1-2`
@@ -66,11 +72,6 @@ function configure_database_postgresql {
        pg_dir=`find /etc/postgresql -name pg_hba.conf|xargs dirname`
        pg_hba=$pg_dir/pg_hba.conf
        pg_conf=$pg_dir/postgresql.conf
-    elif is_suse; then
-        pg_hba=/var/lib/pgsql/data/pg_hba.conf
-        pg_conf=/var/lib/pgsql/data/postgresql.conf
-        # initdb is called when postgresql is first started
-        sudo [ -e $pg_hba ] || start_service postgresql
    else
        exit_distro_not_supported "postgresql configuration"
    fi
@@ -95,7 +96,6 @@ function configure_database_postgresql {

 function install_database_postgresql {
    echo_summary "Installing postgresql"
-    deprecated "Use of postgresql in devstack is deprecated, and will be removed during the Pike cycle"
    local pgpass=$HOME/.pgpass
    if [[ ! -e $pgpass ]]; then
        cat <<EOF > $pgpass
@@ -105,15 +105,17 @@ EOF
    else
        sed -i "s/:root:\w\+/:root:$DATABASE_PASSWORD/" $pgpass
    fi
-    if is_ubuntu; then
-        install_package postgresql
-    elif is_fedora || is_suse; then
-        install_package postgresql-server
-        if is_fedora; then
-            sudo systemctl enable postgresql
+    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
+        if is_ubuntu; then
+            install_package postgresql
+        elif is_fedora; then
+            install_package postgresql-server
+            if is_fedora; then
+                sudo systemctl enable postgresql
+            fi
+        else
+            exit_distro_not_supported "postgresql installation"
        fi
-    else
-        exit_distro_not_supported "postgresql installation"
    fi
 }

@@ -40,12 +40,18 @@ function start_dstat {
    if is_service_enabled peakmem_tracker; then
        die $LINENO "The peakmem_tracker service has been removed, use memory_tracker instead"
    fi
+
+    # To enable file_tracker add:
+    #    enable_service file_tracker
+    # to your localrc
+    run_process file_tracker "$TOP_DIR/tools/file_tracker.sh"
 }

 # stop_dstat() stop dstat process
 function stop_dstat {
    stop_process dstat
    stop_process memory_tracker
+    stop_process file_tracker
 }

 # Restore xtrace
@@ -51,7 +51,7 @@ function start_etcd3 {
    fi
    cmd+=" --listen-client-urls http://$SERVICE_HOST:$ETCD_PORT"
    if [ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]; then
-        cmd+=" --debug"
+        cmd+=" --log-level=debug"
    fi

    local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
@@ -47,10 +47,21 @@ USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE)
 # from CINDER_ENABLED_BACKENDS
 GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
 GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
-# NOTE (abhishekk): For opensuse data files are stored in different directory
-if is_opensuse; then
-    GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/etc/glance
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+    GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance
 fi
+# When Cinder is used as a glance store, you can optionally configure cinder to
+# optimize bootable volume creation by allowing volumes to be cloned directly
+# in the backend instead of transferring data via Glance.  To use this feature,
+# set CINDER_ALLOWED_DIRECT_URL_SCHEMES for cinder.conf and enable
+# GLANCE_SHOW_DIRECT_URL and/or GLANCE_SHOW_MULTIPLE_LOCATIONS for Glance.  The
+# default value for both of these is False, because for some backends they
+# present a grave security risk (though not for Cinder, because all that's
+# exposed is the volume_id where the image data is stored.)  See OSSN-0065 for
+# more information: https://wiki.openstack.org/wiki/OSSN/OSSN-0065
+GLANCE_SHOW_DIRECT_URL=$(trueorfalse False GLANCE_SHOW_DIRECT_URL)
+GLANCE_SHOW_MULTIPLE_LOCATIONS=$(trueorfalse False GLANCE_SHOW_MULTIPLE_LOCATIONS)
+
 # Glance multi-store configuration
 # Boolean flag to enable multiple store configuration for glance
 GLANCE_ENABLE_MULTIPLE_STORES=$(trueorfalse False GLANCE_ENABLE_MULTIPLE_STORES)
@@ -64,13 +75,14 @@ GLANCE_MULTIPLE_FILE_STORES=${GLANCE_MULTIPLE_FILE_STORES:-fast}
 GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:-fast}

 GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache}
+GLANCE_CACHE_DRIVER=${GLANCE_CACHE_DRIVER:-centralized_db}

 # Full Glance functionality requires running in standalone mode. If we are
 # not in uwsgi mode, then we are standalone, otherwise allow separate control.
 if [[ "$WSGI_MODE" != "uwsgi" ]]; then
    GLANCE_STANDALONE=True
 fi
-GLANCE_STANDALONE=${GLANCE_STANDALONE:-True}
+GLANCE_STANDALONE=${GLANCE_STANDALONE:-False}

 # File path for each store specified in GLANCE_MULTIPLE_FILE_STORES, the store
 # identifier will be appended to this path at runtime. If GLANCE_MULTIPLE_FILE_STORES
@@ -84,6 +96,16 @@ GLANCE_STAGING_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_stagi
 GLANCE_TASKS_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_tasks_store}

 GLANCE_USE_IMPORT_WORKFLOW=$(trueorfalse False GLANCE_USE_IMPORT_WORKFLOW)
+GLANCE_ENABLE_QUOTAS=$(trueorfalse True GLANCE_ENABLE_QUOTAS)
+
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# This is used to disable the Image API policies scope and new defaults.
+# By Default, it is True.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+GLANCE_ENFORCE_SCOPE=$(trueorfalse True GLANCE_ENFORCE_SCOPE)
+
+# Flag to disable image format inspection on upload
+GLANCE_ENFORCE_IMAGE_FORMAT=$(trueorfalse True GLANCE_ENFORCE_IMAGE_FORMAT)

 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
@@ -107,6 +129,10 @@ GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT}
 GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 GLANCE_UWSGI=$GLANCE_BIN_DIR/glance-wsgi-api
 GLANCE_UWSGI_CONF=$GLANCE_CONF_DIR/glance-uwsgi.ini
+
+# Glance default limit for Devstack
+GLANCE_LIMIT_IMAGE_SIZE_TOTAL=${GLANCE_LIMIT_IMAGE_SIZE_TOTAL:-1000}
+
 # If wsgi mode is uwsgi run glance under uwsgi, else default to eventlet
 # TODO(mtreinish): Remove the eventlet path here and in all the similar
 # conditionals below after the Pike release
@@ -130,8 +156,9 @@ function is_glance_enabled {
 # cleanup_glance() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
 function cleanup_glance {
-    # delete image files (glance)
-    sudo rm -rf $GLANCE_CACHE_DIR $GLANCE_IMAGE_DIR
+    # delete image files (glance) and all of the glance-remote temporary
+    # storage
+    sudo rm -rf $GLANCE_CACHE_DIR $GLANCE_IMAGE_DIR "${DATA_DIR}/glance-remote"

    # Cleanup multiple stores directories
    if [[ "$GLANCE_ENABLE_MULTIPLE_STORES" == "True" ]]; then
@@ -144,6 +171,7 @@ function cleanup_glance {
        # Cleanup reserved stores directories
        sudo rm -rf $GLANCE_STAGING_DIR $GLANCE_TASKS_DIR
    fi
+    remove_uwsgi_config "$GLANCE_UWSGI_CONF" "glance-wsgi-api"
 }

 # Set multiple cinder store related config options for each of the cinder store
@@ -262,6 +290,38 @@ function configure_glance_store {
    fi
 }

+function configure_glance_quotas {
+
+    # Registered limit resources in keystone are system-specific resources.
+    # Make sure we use a system-scoped token to interact with this API.
+
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL --region $REGION_NAME image_size_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL --region $REGION_NAME image_stage_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit 100 --region $REGION_NAME image_count_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit 100 --region $REGION_NAME image_count_uploading
+
+    # Tell glance to use these limits
+    iniset $GLANCE_API_CONF DEFAULT use_keystone_limits True
+
+    # Configure oslo_limit so it can talk to keystone
+    iniset $GLANCE_API_CONF oslo_limit user_domain_name $SERVICE_DOMAIN_NAME
+    iniset $GLANCE_API_CONF oslo_limit password $SERVICE_PASSWORD
+    iniset $GLANCE_API_CONF oslo_limit username glance
+    iniset $GLANCE_API_CONF oslo_limit auth_type password
+    iniset $GLANCE_API_CONF oslo_limit auth_url $KEYSTONE_SERVICE_URI
+    iniset $GLANCE_API_CONF oslo_limit system_scope all
+    iniset $GLANCE_API_CONF oslo_limit endpoint_id \
+           $(openstack --os-cloud devstack-system-admin endpoint list --service glance -f value -c ID)
+
+    # Allow the glance service user to read quotas
+    openstack --os-cloud devstack-system-admin role add --user glance \
+        --user-domain $SERVICE_DOMAIN_NAME --system all reader
+}
+
 # configure_glance() - Set config files, create data dirs, etc
 function configure_glance {
    sudo install -d -o $STACK_USER $GLANCE_CONF_DIR $GLANCE_METADEF_DIR
@@ -274,18 +334,19 @@ function configure_glance {
    iniset $GLANCE_API_CONF database connection $dburl
    iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_API_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_API_CONF oslo_concurrency lock_path $GLANCE_LOCK_DIR
    iniset $GLANCE_API_CONF paste_deploy flavor keystone+cachemanagement
    configure_keystone_authtoken_middleware $GLANCE_API_CONF glance
    iniset $GLANCE_API_CONF oslo_messaging_notifications driver messagingv2
    iniset_rpc_backend glance $GLANCE_API_CONF
-    if [ "$VIRT_DRIVER" = 'xenserver' ]; then
-        iniset $GLANCE_API_CONF DEFAULT container_formats "ami,ari,aki,bare,ovf,tgz"
-        iniset $GLANCE_API_CONF DEFAULT disk_formats "ami,ari,aki,vhd,raw,iso"
-    fi
    if [ "$VIRT_DRIVER" = 'libvirt' ] && [ "$LIBVIRT_TYPE" = 'parallels' ]; then
        iniset $GLANCE_API_CONF DEFAULT disk_formats "ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,ploop"
    fi
+    # Only use these if you know what you are doing!  See OSSN-0065
+    iniset $GLANCE_API_CONF DEFAULT show_image_direct_url $GLANCE_SHOW_DIRECT_URL
+    iniset $GLANCE_API_CONF DEFAULT show_multiple_locations $GLANCE_SHOW_MULTIPLE_LOCATIONS
+    iniset $GLANCE_API_CONF image_format require_image_format_match $GLANCE_ENFORCE_IMAGE_FORMAT

    # Configure glance_store
    configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES
@@ -338,6 +399,7 @@ function configure_glance {
    iniset $GLANCE_CACHE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $GLANCE_CACHE_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_SERVICE_URI
    iniset $GLANCE_CACHE_CONF DEFAULT admin_tenant_name $SERVICE_PROJECT_NAME
    iniset $GLANCE_CACHE_CONF DEFAULT admin_user glance
@@ -365,11 +427,27 @@ function configure_glance {

    if [[ "$GLANCE_STANDALONE" == False ]]; then
        write_local_uwsgi_http_config "$GLANCE_UWSGI_CONF" "$GLANCE_UWSGI" "/image"
+        # Grab our uwsgi listen address and use that to fill out our
+        # worker_self_reference_url config
+        iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url \
+               $(awk '-F= ' '/^http-socket/ { print "http://"$2}' \
+                    $GLANCE_UWSGI_CONF)
    else
        write_local_proxy_http_config glance "http://$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT_INT" "/image"
        iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
        iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
        iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
+        iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url $GLANCE_URL
+    fi
+
+    if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $GLANCE_API_CONF oslo_policy enforce_scope true
+        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults true
+        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac true
+    else
+        iniset $GLANCE_API_CONF oslo_policy enforce_scope false
+        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults false
+        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac false
    fi
 }

@@ -401,6 +479,11 @@ function create_glance_accounts {
        service_domain_id=$(get_or_create_domain $SERVICE_DOMAIN_NAME)
        iniset $GLANCE_SWIFT_STORE_CONF ref1 project_domain_id $service_domain_id
        iniset $GLANCE_SWIFT_STORE_CONF ref1 user_domain_id $service_domain_id
+
+        if [[ "$GLANCE_ENABLE_QUOTAS" = True ]]; then
+            configure_glance_quotas
+        fi
+
    fi
 }

@@ -460,6 +543,76 @@ function install_glance {
    setup_develop $GLANCE_DIR
 }

+# glance_remote_conf() - Return the path to an alternate config file for
+#                        the remote glance clone
+function glance_remote_conf {
+    echo $(dirname "${GLANCE_CONF_DIR}")/glance-remote/$(basename "$1")
+}
+
+# start_glance_remote_clone() - Clone the regular glance api worker
+function start_glance_remote_clone {
+    local glance_remote_conf_dir glance_remote_port remote_data
+    local glance_remote_uwsgi venv
+
+    glance_remote_conf_dir="$(glance_remote_conf "")"
+    glance_remote_port=$(get_random_port)
+    glance_remote_uwsgi="$(glance_remote_conf $GLANCE_UWSGI_CONF)"
+
+    # Clone the existing ready-to-go glance-api setup
+    sudo rm -Rf "$glance_remote_conf_dir"
+    sudo cp -r "$GLANCE_CONF_DIR" "$glance_remote_conf_dir"
+    sudo chown $STACK_USER -R "$glance_remote_conf_dir"
+
+    # Point this worker at different data dirs
+    remote_data="${DATA_DIR}/glance-remote"
+    mkdir -p $remote_data/os_glance_tasks_store \
+          "${remote_data}/os_glance_staging_store"
+    iniset $(glance_remote_conf "$GLANCE_API_CONF") os_glance_staging_store \
+           filesystem_store_datadir "${remote_data}/os_glance_staging_store"
+    iniset $(glance_remote_conf "$GLANCE_API_CONF") os_glance_tasks_store \
+           filesystem_store_datadir "${remote_data}/os_glance_tasks_store"
+
+    # Point this worker to use different cache dir
+    mkdir -p "$remote_data/cache"
+    iniset $(glance_remote_conf "$GLANCE_API_CONF") DEFAULT \
+           image_cache_dir "${remote_data}/cache"
+
+    # Change our uwsgi to our new port
+    sed -ri "s/^(http-socket.*):[0-9]+/\1:$glance_remote_port/" \
+        "$glance_remote_uwsgi"
+
+    # Update the self-reference url with our new port
+    iniset $(glance_remote_conf $GLANCE_API_CONF) DEFAULT \
+           worker_self_reference_url \
+           $(awk '-F= ' '/^http-socket/ { print "http://"$2 }' \
+                    "$glance_remote_uwsgi")
+
+    # We need to create the systemd service for the clone, but then
+    # change it to include an Environment line to point the WSGI app
+    # at the alternate config directory.
+    if [[ "$GLOBAL_VENV" == True ]]; then
+        venv="--venv $DEVSTACK_VENV"
+    fi
+    write_uwsgi_user_unit_file devstack@g-api-r.service "$(which uwsgi) \
+                               --procname-prefix \
+                               glance-api-remote \
+                               --ini $glance_remote_uwsgi \
+                               $venv" \
+                               "" "$STACK_USER"
+    iniadd -sudo ${SYSTEMD_DIR}/devstack@g-api-r.service \
+           "Service" "Environment" \
+           "OS_GLANCE_CONFIG_DIR=$glance_remote_conf_dir"
+
+    # Reload and restart with the new config
+    $SYSTEMCTL daemon-reload
+    $SYSTEMCTL restart devstack@g-api-r
+
+    get_or_create_service glance_remote image_remote "Alternate glance"
+    get_or_create_endpoint image_remote $REGION_NAME \
+                $(awk '-F= ' '/^http-socket/ { print "http://"$2 }' \
+                    $glance_remote_uwsgi)
+}
+
 # start_glance() - Start running processes
 function start_glance {
    local service_protocol=$GLANCE_SERVICE_PROTOCOL
@@ -475,6 +628,11 @@ function start_glance {
        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR"
    fi

+    if is_service_enabled g-api-r; then
+        echo "Starting the g-api-r clone service..."
+        start_glance_remote_clone
+    fi
+
    echo "Waiting for g-api ($GLANCE_SERVICE_HOST) to start..."
    if ! wait_for_service $SERVICE_TIMEOUT $GLANCE_URL; then
        die $LINENO "g-api did not start"
@@ -484,6 +642,7 @@ function start_glance {
 # stop_glance() - Stop running processes
 function stop_glance {
    stop_process g-api
+    stop_process g-api-r
 }

 # Restore xtrace
@@ -109,12 +109,21 @@ function configure_horizon {
        _horizon_config_set $local_settings "" OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT "True"
    fi

+    if is_service_enabled c-bak; then
+        _horizon_config_set $local_settings OPENSTACK_CINDER_FEATURES enable_backup "True"
+    fi
+
    # Create an empty directory that apache uses as docroot
    sudo mkdir -p $HORIZON_DIR/.blackhole

    local horizon_conf
    horizon_conf=$(apache_site_config_for horizon)

+    local wsgi_venv_config=""
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV"
+    fi
+
    # Configure apache to run horizon
    # Set up the django horizon application to serve via apache/wsgi
    sudo sh -c "sed -e \"
@@ -124,12 +133,13 @@ function configure_horizon {
        s,%APACHE_NAME%,$APACHE_NAME,g;
        s,%DEST%,$DEST,g;
        s,%WEBROOT%,$HORIZON_APACHE_ROOT,g;
+        s,%WSGIPYTHONHOME%,$wsgi_venv_config,g;
    \" $FILES/apache-horizon.template >$horizon_conf"

    if is_ubuntu; then
        disable_apache_site 000-default
        sudo touch $horizon_conf
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        : # nothing to do
    else
        exit_distro_not_supported "horizon apache configuration"
@@ -163,6 +173,10 @@ function install_horizon {
    # Apache installation, because we mark it NOPRIME
    install_apache_wsgi

+    # Install the memcache library so that horizon can use memcached as its
+    # cache backend
+    pip_install_gr pymemcache
+
    git_clone $HORIZON_REPO $HORIZON_DIR $HORIZON_BRANCH
 }

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Kernel Samepage Merging (KSM)
+# -----------------------------
+
+# Processes that mark their memory as mergeable can share identical memory
+# pages if KSM is enabled. This is particularly useful for nova + libvirt
+# backends but any other setup that marks its memory as mergeable can take
+# advantage. The drawback is there is higher cpu load; however, we tend to
+# be memory bound not cpu bound so enable KSM by default but allow people
+# to opt out if the CPU time is more important to them.
+ENABLE_KSM=$(trueorfalse True ENABLE_KSM)
+ENABLE_KSMTUNED=$(trueorfalse True ENABLE_KSMTUNED)
+function configure_ksm {
+    if [[ $ENABLE_KSMTUNED == "True" ]] ; then
+        install_package "ksmtuned"
+    fi
+    if [[ -f /sys/kernel/mm/ksm/run ]] ; then
+        echo $(bool_to_int ENABLE_KSM) | sudo tee /sys/kernel/mm/ksm/run
+    fi
+}
+
+# Compressed swap (ZSWAP)
+#------------------------
+
+# as noted in the kernel docs https://docs.kernel.org/admin-guide/mm/zswap.html
+# Zswap is a lightweight compressed cache for swap pages.
+# It takes pages that are in the process of being swapped out and attempts
+# to compress them into a dynamically allocated RAM-based memory pool.
+# zswap basically trades CPU cycles for potentially reduced swap I/O.
+# This trade-off can also result in a significant performance improvement
+# if reads from the compressed cache are faster than reads from a swap device.
+
+ENABLE_ZSWAP=$(trueorfalse False ENABLE_ZSWAP)
+# lz4 is very fast although it does not have the best compression
+# zstd has much better compression but more latency
+ZSWAP_COMPRESSOR=${ZSWAP_COMPRESSOR:="lz4"}
+ZSWAP_ZPOOL=${ZSWAP_ZPOOL:="z3fold"}
+function configure_zswap {
+    if [[ $ENABLE_ZSWAP == "True" ]] ; then
+        # Centos 9 stream seems to only support enabling but not run time
+        # tuning so dont try to choose better default on centos
+        if is_ubuntu; then
+            echo ${ZSWAP_COMPRESSOR} | sudo tee /sys/module/zswap/parameters/compressor
+            echo ${ZSWAP_ZPOOL} | sudo tee /sys/module/zswap/parameters/zpool
+        fi
+        echo 1 | sudo tee /sys/module/zswap/parameters/enabled
+        # print curent zswap kernel config
+        sudo grep -R . /sys/module/zswap/parameters || /bin/true
+    fi
+}
+
+ENABLE_SYSCTL_MEM_TUNING=$(trueorfalse False ENABLE_SYSCTL_MEM_TUNING)
+function configure_sysctl_mem_parmaters {
+    if [[ $ENABLE_SYSCTL_MEM_TUNING == "True" ]] ; then
+        # defer write when memory is available
+        sudo sysctl -w vm.dirty_ratio=60
+        sudo sysctl -w vm.dirty_background_ratio=10
+        sudo sysctl -w vm.vfs_cache_pressure=50
+        # assume swap is compressed so on new kernels
+        # give it equal priority as page cache which is
+        # uncompressed. on kernels < 5.8 the max is 100
+        # not 200 so it will strongly prefer swapping.
+        sudo sysctl -w vm.swappiness=100
+        sudo grep -R . /proc/sys/vm/  || /bin/true
+    fi
+}
+
+function configure_host_mem {
+    configure_zswap
+    configure_ksm
+    configure_sysctl_mem_parmaters
+}
+
+ENABLE_SYSCTL_NET_TUNING=$(trueorfalse False ENABLE_SYSCTL_NET_TUNING)
+function configure_sysctl_net_parmaters {
+    if [[ $ENABLE_SYSCTL_NET_TUNING == "True" ]] ; then
+        # detect dead TCP connections after 120 seconds
+        sudo sysctl -w net.ipv4.tcp_keepalive_time=60
+        sudo sysctl -w net.ipv4.tcp_keepalive_intvl=10
+        sudo sysctl -w net.ipv4.tcp_keepalive_probes=6
+        # reudce network latency for new connections
+        sudo sysctl -w net.ipv4.tcp_fastopen=3
+        # print tcp options
+        sudo grep -R . /proc/sys/net/ipv4/tcp* || /bin/true
+        # disable qos by default
+        sudo sysctl -w net.core.default_qdisc=pfifo_fast
+    fi
+}
+
+function configure_host_net {
+    configure_sysctl_net_parmaters
+}
+
+function tune_host {
+    configure_host_mem
+    configure_host_net
+}
@@ -31,7 +31,7 @@ function install_infra {
    local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv"
    [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV
    # We don't care about testing git pbr in the requirements venv.
-    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr
+    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr setuptools
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR

    # Unset the PIP_VIRTUAL_ENV so that PBR does not end up trapped
@@ -9,7 +9,6 @@
 # - ``tls`` file
 # - ``DEST``, ``STACK_USER``
 # - ``FILES``
-# - ``IDENTITY_API_VERSION``
 # - ``BASE_SQL_CONN``
 # - ``SERVICE_HOST``, ``SERVICE_PROTOCOL``
 # - ``S3_SERVICE_PORT`` (template backend only)
@@ -50,9 +49,7 @@ fi
 KEYSTONE_CONF_DIR=${KEYSTONE_CONF_DIR:-/etc/keystone}
 KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
 KEYSTONE_PUBLIC_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-public.ini
-KEYSTONE_ADMIN_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-admin.ini
 KEYSTONE_PUBLIC_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-public
-KEYSTONE_ADMIN_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-admin

 # KEYSTONE_DEPLOY defines how keystone is deployed, allowed values:
 # - mod_wsgi : Run keystone under Apache HTTPd mod_wsgi
@@ -81,21 +78,12 @@ KEYSTONE_RESOURCE_BACKEND=${KEYSTONE_RESOURCE_BACKEND:-sql}
 KEYSTONE_TOKEN_FORMAT=${KEYSTONE_TOKEN_FORMAT:-fernet}
 KEYSTONE_TOKEN_FORMAT=$(echo ${KEYSTONE_TOKEN_FORMAT} | tr '[:upper:]' '[:lower:]')

-# Set Keystone interface configuration
-KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
-KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
-KEYSTONE_AUTH_PORT_INT=${KEYSTONE_AUTH_PORT_INT:-35358}
-KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
-
 # Public facing bits
 KEYSTONE_SERVICE_HOST=${KEYSTONE_SERVICE_HOST:-$SERVICE_HOST}
 KEYSTONE_SERVICE_PORT=${KEYSTONE_SERVICE_PORT:-5000}
 KEYSTONE_SERVICE_PORT_INT=${KEYSTONE_SERVICE_PORT_INT:-5001}
 KEYSTONE_SERVICE_PROTOCOL=${KEYSTONE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}

-# Bind hosts
-KEYSTONE_ADMIN_BIND_HOST=${KEYSTONE_ADMIN_BIND_HOST:-$KEYSTONE_SERVICE_HOST}
-
 # Set the project for service accounts in Keystone
 SERVICE_DOMAIN_NAME=${SERVICE_DOMAIN_NAME:-Default}
 SERVICE_PROJECT_NAME=${SERVICE_PROJECT_NAME:-service}
@@ -106,7 +94,6 @@ SERVICE_TENANT_NAME=${SERVICE_PROJECT_NAME:-service}

 # if we are running with SSL use https protocols
 if is_service_enabled tls-proxy; then
-    KEYSTONE_AUTH_PROTOCOL="https"
    KEYSTONE_SERVICE_PROTOCOL="https"
 fi

@@ -134,6 +121,15 @@ KEYSTONE_PASSWORD_HASH_ROUNDS=${KEYSTONE_PASSWORD_HASH_ROUNDS:-4}
 # Cache settings
 KEYSTONE_ENABLE_CACHE=${KEYSTONE_ENABLE_CACHE:-True}

+# Whether to create a keystone admin endpoint for legacy applications
+KEYSTONE_ADMIN_ENDPOINT=$(trueorfalse False KEYSTONE_ADMIN_ENDPOINT)
+
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the Identity API policies to start checking the scope of token. By Default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+KEYSTONE_ENFORCE_SCOPE=$(trueorfalse False KEYSTONE_ENFORCE_SCOPE)
+
 # Functions
 # ---------

@@ -154,11 +150,8 @@ function cleanup_keystone {
        sudo rm -f $(apache_site_config_for keystone)
    else
        stop_process "keystone"
-        # TODO: remove admin at pike-2
-        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI"
-        remove_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI"
+        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "keystone-wsgi-public"
        sudo rm -f $(apache_site_config_for keystone-wsgi-public)
-        sudo rm -f $(apache_site_config_for keystone-wsgi-admin)
    fi
 }

@@ -171,12 +164,10 @@ function _config_keystone_apache_wsgi {
    local keystone_certfile=""
    local keystone_keyfile=""
    local keystone_service_port=$KEYSTONE_SERVICE_PORT
-    local keystone_auth_port=$KEYSTONE_AUTH_PORT
    local venv_path=""

    if is_service_enabled tls-proxy; then
        keystone_service_port=$KEYSTONE_SERVICE_PORT_INT
-        keystone_auth_port=$KEYSTONE_AUTH_PORT_INT
    fi
    if [[ ${USE_VENV} = True ]]; then
        venv_path="python-path=${PROJECT_VENV["keystone"]}/lib/$(python_version)/site-packages"
@@ -185,7 +176,6 @@ function _config_keystone_apache_wsgi {
    sudo cp $FILES/apache-keystone.template $keystone_apache_conf
    sudo sed -e "
        s|%PUBLICPORT%|$keystone_service_port|g;
-        s|%ADMINPORT%|$keystone_auth_port|g;
        s|%APACHE_NAME%|$APACHE_NAME|g;
        s|%SSLLISTEN%|$keystone_ssl_listen|g;
        s|%SSLENGINE%|$keystone_ssl|g;
@@ -223,22 +213,17 @@ function configure_keystone {
    iniset_rpc_backend keystone $KEYSTONE_CONF oslo_messaging_notifications

    local service_port=$KEYSTONE_SERVICE_PORT
-    local auth_port=$KEYSTONE_AUTH_PORT

    if is_service_enabled tls-proxy; then
        # Set the service ports for a proxy to take the originals
        service_port=$KEYSTONE_SERVICE_PORT_INT
-        auth_port=$KEYSTONE_AUTH_PORT_INT
    fi

-    # Override the endpoints advertised by keystone (the public_endpoint and
-    # admin_endpoint) so that clients use the correct endpoint. By default, the
-    # keystone server uses the public_port and admin_port which isn't going to
-    # work when you want to use a different port (in the case of proxy), or you
-    # don't want the port (in the case of putting keystone on a path in
-    # apache).
+    # Override the endpoints advertised by keystone so that clients use the correct
+    # endpoint. By default, the keystone server uses the public_port which isn't
+    # going to work when you want to use a different port (in the case of proxy),
+    # or you don't want the port (in the case of putting keystone on a path in apache).
    iniset $KEYSTONE_CONF DEFAULT public_endpoint $KEYSTONE_SERVICE_URI
-    iniset $KEYSTONE_CONF DEFAULT admin_endpoint $KEYSTONE_AUTH_URI

    if [[ "$KEYSTONE_TOKEN_FORMAT" != "" ]]; then
        iniset $KEYSTONE_CONF token provider $KEYSTONE_TOKEN_FORMAT
@@ -261,7 +246,6 @@ function configure_keystone {
        _config_keystone_apache_wsgi
    else # uwsgi
        write_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI" "/identity"
-        write_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI" "/identity_admin"
    fi

    iniset $KEYSTONE_CONF DEFAULT max_token_size 16384
@@ -281,6 +265,16 @@ function configure_keystone {
        iniset $KEYSTONE_CONF security_compliance lockout_duration $KEYSTONE_LOCKOUT_DURATION
        iniset $KEYSTONE_CONF security_compliance unique_last_password_count $KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT
    fi
+
+    iniset $KEYSTONE_CONF oslo_policy policy_file policy.yaml
+
+    if [[ "$KEYSTONE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $KEYSTONE_CONF oslo_policy enforce_scope true
+        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults true
+    else
+        iniset $KEYSTONE_CONF oslo_policy enforce_scope false
+        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults false
+    fi
 }

 # create_keystone_accounts() - Sets up common required keystone accounts
@@ -303,40 +297,48 @@ function configure_keystone {
 # admins               admin            admin                 admin
 # nonadmins            demo, alt_demo   member, anotherrole   demo, alt_demo

+# System               User            Roles
+# ------------------------------------------------------------------
+# all                  admin           admin
+# all                  system_reader   reader
+# all                  system_member   member
+

 # Migrated from keystone_data.sh
 function create_keystone_accounts {

    # The keystone bootstrapping process (performed via keystone-manage
-    # bootstrap) creates an admin user, admin role, member role, and admin
+    # bootstrap) creates an admin user and an admin
    # project. As a sanity check we exercise the CLI to retrieve the IDs for
    # these values.
    local admin_project
    admin_project=$(openstack project show "admin" -f value -c id)
    local admin_user
    admin_user=$(openstack user show "admin" -f value -c id)
+    # These roles are also created during bootstrap but we don't need their IDs
    local admin_role="admin"
    local member_role="member"
+    local reader_role="reader"

-    get_or_add_user_domain_role $admin_role $admin_user default
+    async_run ks-domain-role get_or_add_user_domain_role $admin_role $admin_user default

    # Create service project/role
    get_or_create_domain "$SERVICE_DOMAIN_NAME"
-    get_or_create_project "$SERVICE_PROJECT_NAME" "$SERVICE_DOMAIN_NAME"
+    async_run ks-project get_or_create_project "$SERVICE_PROJECT_NAME" "$SERVICE_DOMAIN_NAME"

    # Service role, so service users do not have to be admins
-    get_or_create_role service
+    async_run ks-service get_or_create_role service

    # The ResellerAdmin role is used by Nova and Ceilometer so we need to keep it.
    # The admin role in swift allows a user to act as an admin for their project,
    # but ResellerAdmin is needed for a user to act as any project. The name of this
    # role is also configurable in swift-proxy.conf
-    get_or_create_role ResellerAdmin
+    async_run ks-reseller get_or_create_role ResellerAdmin

    # another_role demonstrates that an arbitrary role may be created and used
    # TODO(sleepsonthefloor): show how this can be used for rbac in the future!
    local another_role="anotherrole"
-    get_or_create_role $another_role
+    async_run ks-anotherrole get_or_create_role $another_role

    # invisible project - admin can't see this one
    local invis_project
@@ -349,21 +351,55 @@ function create_keystone_accounts {
    demo_user=$(get_or_create_user "demo" \
        "$ADMIN_PASSWORD" "default" "demo@example.com")

-    get_or_add_user_project_role $member_role $demo_user $demo_project
-    get_or_add_user_project_role $admin_role $admin_user $demo_project
-    get_or_add_user_project_role $another_role $demo_user $demo_project
-    get_or_add_user_project_role $member_role $demo_user $invis_project
+    async_wait ks-{domain-role,domain,project,service,reseller,anotherrole}

-    # alt_demo
+    async_run ks-demo-member get_or_add_user_project_role $member_role $demo_user $demo_project
+
+    async_run ks-demo-admin get_or_add_user_project_role $admin_role $admin_user $demo_project
+    async_run ks-demo-another get_or_add_user_project_role $another_role $demo_user $demo_project
+    async_run ks-demo-invis get_or_add_user_project_role $member_role $demo_user $invis_project
+
+    # Create a user to act as a reader on project demo
+    local demo_reader
+    demo_reader=$(get_or_create_user "demo_reader" \
+        "$ADMIN_PASSWORD" "default" "demo_reader@example.com")
+
+    async_run ks-demo-reader get_or_add_user_project_role $reader_role $demo_reader $demo_project
+
+    # Create a different project called alt_demo
    local alt_demo_project
    alt_demo_project=$(get_or_create_project "alt_demo" default)
+    # Create a user to act as member, admin and anotherrole on project alt_demo
    local alt_demo_user
    alt_demo_user=$(get_or_create_user "alt_demo" \
        "$ADMIN_PASSWORD" "default" "alt_demo@example.com")

-    get_or_add_user_project_role $member_role $alt_demo_user $alt_demo_project
-    get_or_add_user_project_role $admin_role $admin_user $alt_demo_project
-    get_or_add_user_project_role $another_role $alt_demo_user $alt_demo_project
+    async_run ks-alt-admin get_or_add_user_project_role $admin_role $alt_demo_user $alt_demo_project
+    async_run ks-alt-another get_or_add_user_project_role $another_role $alt_demo_user $alt_demo_project
+
+    # Create another user to act as a member on project alt_demo
+    local alt_demo_member
+    alt_demo_member=$(get_or_create_user "alt_demo_member" \
+        "$ADMIN_PASSWORD" "default" "alt_demo_member@example.com")
+    async_run ks-alt-member-user get_or_add_user_project_role $member_role $alt_demo_member $alt_demo_project
+
+    # Create another user to act as a reader on project alt_demo
+    local alt_demo_reader
+    alt_demo_reader=$(get_or_create_user "alt_demo_reader" \
+        "$ADMIN_PASSWORD" "default" "alt_demo_reader@example.com")
+    async_run ks-alt-reader-user get_or_add_user_project_role $reader_role $alt_demo_reader $alt_demo_project
+
+    # Create two users, give one the member role on the system and the other the
+    # reader role on the system. These two users model system-member and
+    # system-reader personas. The admin user already has the admin role on the
+    # system and we can re-use this user as a system-admin.
+    system_member_user=$(get_or_create_user "system_member" \
+        "$ADMIN_PASSWORD" "default" "system_member@example.com")
+    async_run ks-system-member get_or_add_user_system_role $member_role $system_member_user "all"
+
+    system_reader_user=$(get_or_create_user "system_reader" \
+        "$ADMIN_PASSWORD" "default" "system_reader@example.com")
+    async_run ks-system-reader get_or_add_user_system_role $reader_role $system_reader_user "all"

    # groups
    local admin_group
@@ -373,11 +409,16 @@ function create_keystone_accounts {
    non_admin_group=$(get_or_create_group "nonadmins" \
        "default" "non-admin group")

-    get_or_add_group_project_role $member_role $non_admin_group $demo_project
-    get_or_add_group_project_role $another_role $non_admin_group $demo_project
-    get_or_add_group_project_role $member_role $non_admin_group $alt_demo_project
-    get_or_add_group_project_role $another_role $non_admin_group $alt_demo_project
-    get_or_add_group_project_role $admin_role $admin_group $admin_project
+    async_run ks-group-memberdemo get_or_add_group_project_role $member_role $non_admin_group $demo_project
+    async_run ks-group-anotherdemo get_or_add_group_project_role $another_role $non_admin_group $demo_project
+    async_run ks-group-memberalt get_or_add_group_project_role $member_role $non_admin_group $alt_demo_project
+    async_run ks-group-anotheralt get_or_add_group_project_role $another_role $non_admin_group $alt_demo_project
+    async_run ks-group-admin get_or_add_group_project_role $admin_role $admin_group $admin_project
+
+    async_wait ks-demo-{member,admin,another,invis,reader}
+    async_wait ks-alt-{admin,another,member-user,reader-user}
+    async_wait ks-system-{member,reader}
+    async_wait ks-group-{memberdemo,anotherdemo,memberalt,anotheralt,admin}

    if is_service_enabled ldap; then
        create_ldap_domain
@@ -512,7 +553,7 @@ function install_keystone {
 function start_keystone {
    # Get right service port for testing
    local service_port=$KEYSTONE_SERVICE_PORT
-    local auth_protocol=$KEYSTONE_AUTH_PROTOCOL
+    local auth_protocol=$KEYSTONE_SERVICE_PROTOCOL
    if is_service_enabled tls-proxy; then
        service_port=$KEYSTONE_SERVICE_PORT_INT
        auth_protocol="http"
@@ -531,7 +572,7 @@ function start_keystone {
    # unencryted traffic at this point.
    # If running in Apache, use the path rather than port.

-    local service_uri=$auth_protocol://$KEYSTONE_SERVICE_HOST/identity/v$IDENTITY_API_VERSION/
+    local service_uri=$auth_protocol://$KEYSTONE_SERVICE_HOST/identity/v3/

    if ! wait_for_service $SERVICE_TIMEOUT $service_uri; then
        die $LINENO "keystone did not start"
@@ -540,7 +581,6 @@ function start_keystone {
    # Start proxies if enabled
    if is_service_enabled tls-proxy; then
        start_tls_proxy keystone-service '*' $KEYSTONE_SERVICE_PORT $KEYSTONE_SERVICE_HOST $KEYSTONE_SERVICE_PORT_INT
-        start_tls_proxy keystone-auth '*' $KEYSTONE_AUTH_PORT $KEYSTONE_AUTH_HOST $KEYSTONE_AUTH_PORT_INT
    fi

    # (re)start memcached to make sure we have a clean memcache.
@@ -561,11 +601,8 @@ function stop_keystone {
 # This function uses the following GLOBAL variables:
 # - ``KEYSTONE_BIN_DIR``
 # - ``ADMIN_PASSWORD``
-# - ``IDENTITY_API_VERSION``
 # - ``REGION_NAME``
-# - ``KEYSTONE_SERVICE_PROTOCOL``
-# - ``KEYSTONE_SERVICE_HOST``
-# - ``KEYSTONE_SERVICE_PORT``
+# - ``KEYSTONE_SERVICE_URI``
 function bootstrap_keystone {
    $KEYSTONE_BIN_DIR/keystone-manage bootstrap \
        --bootstrap-username admin \
@@ -574,8 +611,16 @@ function bootstrap_keystone {
        --bootstrap-role-name admin \
        --bootstrap-service-name keystone \
        --bootstrap-region-id "$REGION_NAME" \
-        --bootstrap-admin-url "$KEYSTONE_AUTH_URI" \
        --bootstrap-public-url "$KEYSTONE_SERVICE_URI"
+    if [ "$KEYSTONE_ADMIN_ENDPOINT" == "True" ]; then
+        openstack endpoint create --region "$REGION_NAME" \
+            --os-username admin \
+            --os-user-domain-id default \
+            --os-password "$ADMIN_PASSWORD" \
+            --os-project-name admin \
+            --os-project-domain-id default \
+            keystone admin "$KEYSTONE_SERVICE_URI"
+    fi
 }

 # create_ldap_domain() - Create domain file and initialize domain with a user
@@ -33,16 +33,12 @@ LDAP_SERVICE_NAME=slapd

 if is_ubuntu; then
    LDAP_OLCDB_NUMBER=1
+    LDAP_OLCDB_TYPE=mdb
    LDAP_ROOTPW_COMMAND=replace
 elif is_fedora; then
    LDAP_OLCDB_NUMBER=2
+    LDAP_OLCDB_TYPE=hdb
    LDAP_ROOTPW_COMMAND=add
-elif is_suse; then
-    # SUSE has slappasswd in /usr/sbin/
-    PATH=$PATH:/usr/sbin/
-    LDAP_OLCDB_NUMBER=1
-    LDAP_ROOTPW_COMMAND=add
-    LDAP_SERVICE_NAME=ldap
 fi


@@ -56,6 +52,7 @@ function _ldap_varsubst {
    local slappass=$2
    sed -e "
        s|\${LDAP_OLCDB_NUMBER}|$LDAP_OLCDB_NUMBER|
+        s|\${LDAP_OLCDB_TYPE}|$LDAP_OLCDB_TYPE|
        s|\${SLAPPASS}|$slappass|
        s|\${LDAP_ROOTPW_COMMAND}|$LDAP_ROOTPW_COMMAND|
        s|\${BASE_DC}|$LDAP_BASE_DC|
@@ -72,8 +69,6 @@ function cleanup_ldap {
        sudo rm -rf /etc/ldap/ldap.conf /var/lib/ldap
    elif is_fedora; then
        sudo rm -rf /etc/openldap /var/lib/ldap
-    elif is_suse; then
-        sudo rm -rf /var/lib/ldap
    fi
 }

@@ -122,11 +117,6 @@ function install_ldap {
        configure_ldap
    elif is_fedora; then
        start_ldap
-    elif is_suse; then
-        _ldap_varsubst $FILES/ldap/suse-base-config.ldif.in >$tmp_ldap_dir/suse-base-config.ldif
-        sudo slapadd -F /etc/openldap/slapd.d/ -bcn=config -l $tmp_ldap_dir/suse-base-config.ldif
-        sudo sed -i '/^OPENLDAP_START_LDAPI=/s/"no"/"yes"/g' /etc/sysconfig/openldap
-        start_ldap
    fi

    echo "LDAP_PASSWORD is $LDAP_PASSWORD"
@@ -157,7 +147,7 @@ function configure_ldap {
    slapd slapd/dump_database_destdir string /var/backups/slapd-VERSION
    slapd slapd/domain string Users
    slapd shared/organization string $LDAP_DOMAIN
-    slapd slapd/backend string HDB
+    slapd slapd/backend string ${LDAP_OLCDB_TYPE^^}
    slapd slapd/purge_database boolean true
    slapd slapd/move_old_database boolean true
    slapd slapd/allow_ldap_v2 boolean false
@@ -38,6 +38,7 @@ GITDIR["oslo.config"]=$DEST/oslo.config
 GITDIR["oslo.context"]=$DEST/oslo.context
 GITDIR["oslo.db"]=$DEST/oslo.db
 GITDIR["oslo.i18n"]=$DEST/oslo.i18n
+GITDIR["oslo.limit"]=$DEST/oslo.limit
 GITDIR["oslo.log"]=$DEST/oslo.log
 GITDIR["oslo.messaging"]=$DEST/oslo.messaging
 GITDIR["oslo.middleware"]=$DEST/oslo.middleware
@@ -59,6 +60,7 @@ GITDIR["tooz"]=$DEST/tooz
 # Non oslo libraries are welcomed below as well, this prevents
 # duplication of this code.
 GITDIR["os-brick"]=$DEST/os-brick
+GITDIR["os-resource-classes"]=$DEST/os-resource-classes
 GITDIR["os-traits"]=$DEST/os-traits

 # Support entry points installation of console scripts
@@ -101,6 +103,7 @@ function install_libs {
    _install_lib_from_source "oslo.context"
    _install_lib_from_source "oslo.db"
    _install_lib_from_source "oslo.i18n"
+    _install_lib_from_source "oslo.limit"
    _install_lib_from_source "oslo.log"
    _install_lib_from_source "oslo.messaging"
    _install_lib_from_source "oslo.middleware"
@@ -122,6 +125,7 @@ function install_libs {
    #
    # os-traits for nova
    _install_lib_from_source "os-brick"
+    _install_lib_from_source "os-resource-classes"
    _install_lib_from_source "os-traits"
    #
    # python client libraries we might need from git can go here
@@ -53,28 +53,10 @@ function _remove_lvm_volume_group {
    sudo vgremove -f $vg
 }

-# _clean_lvm_backing_file() removes the backing file of the
-# volume group
-#
-# Usage: _clean_lvm_backing_file() $backing_file
-function _clean_lvm_backing_file {
-    local backing_file=$1
-
-    # If the backing physical device is a loop device, it was probably setup by DevStack
-    if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
-        local vg_dev
-        vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}')
-        if [[ -n "$vg_dev" ]]; then
-            sudo losetup -d $vg_dev
-        fi
-        rm -f $backing_file
-    fi
-}
-
 # clean_lvm_volume_group() cleans up the volume group and removes the
 # backing file
 #
-# Usage: clean_lvm_volume_group $vg
+# Usage: clean_lvm_volume_group() $vg
 function clean_lvm_volume_group {
    local vg=$1

@@ -83,11 +65,22 @@ function clean_lvm_volume_group {
    # if there is no logical volume left, it's safe to attempt a cleanup
    # of the backing file
    if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
-        _clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX
+        local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX
+
+        if [[ -n "$vg$BACKING_FILE_SUFFIX" ]] && \
+           [[ -e "/etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service" ]]; then
+            sudo systemctl disable --now $vg$BACKING_FILE_SUFFIX.service
+            sudo rm -f /etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service
+            sudo systemctl daemon-reload
+        fi
+
+        # If the backing physical device is a loop device, it was probably setup by DevStack
+        if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
+            rm -f $backing_file
+        fi
    fi
 }

-
 # _create_lvm_volume_group creates default volume group
 #
 # Usage: _create_lvm_volume_group() $vg $size
@@ -106,8 +99,20 @@ function _create_lvm_volume_group {
            directio="--direct-io=on"
        fi

+        # Only create systemd service if it doesn't already exists
+        if [[ ! -e "/etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service" ]]; then
+            sed -e "
+                s|%DIRECTIO%|${directio}|g;
+                s|%BACKING_FILE%|${backing_file}|g;
+            " $FILES/lvm-backing-file.template | sudo tee \
+                /etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service
+
+            sudo systemctl daemon-reload
+            sudo systemctl enable --now $vg$BACKING_FILE_SUFFIX.service
+        fi
+
        local vg_dev
-        vg_dev=$(sudo losetup -f --show $directio $backing_file)
+        vg_dev=$(sudo losetup --associated $backing_file -O NAME -n)

        # Only create volume group if it doesn't already exist
        if ! sudo vgs $vg; then
@@ -124,19 +129,25 @@ function init_lvm_volume_group {
    local vg=$1
    local size=$2

-    # Start the tgtd service on Fedora and SUSE if tgtadm is used
-    if  is_fedora || is_suse  && [[ "$CINDER_ISCSI_HELPER" = "tgtadm" ]]; then
+    # Start the tgtd service on Fedora if tgtadm is used
+    if  is_fedora; then
        start_service tgtd
    fi

    # Start with a clean volume group
    _create_lvm_volume_group $vg $size

-    # Remove iscsi targets
-    if [ "$CINDER_ISCSI_HELPER" = "lioadm" ]; then
-        sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
-    else
-        sudo tgtadm --op show --mode target | awk '/Target/ {print $3}' | sudo xargs -r -n1 tgt-admin --delete
+    if is_service_enabled cinder; then
+        # Remove iscsi targets
+        if [ "$CINDER_TARGET_HELPER" = "lioadm" ]; then
+            sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
+        elif [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
+            sudo tgtadm --op show --mode target | awk '/Target/ {print $3}' | sudo xargs -r -n1 tgt-admin --delete
+        elif [ "$CINDER_TARGET_HELPER" = "nvmet" ]; then
+            # If we don't disconnect everything vgremove will block
+            sudo nvme disconnect-all
+            sudo nvmetcli clear
+        fi
    fi
    _clean_lvm_volume_group $vg
 }
@@ -189,7 +200,7 @@ function set_lvm_filter {
    filter_string=$filter_string$filter_suffix

    clean_lvm_filter
-    sudo sed -i "/# global_filter = \[*\]/a\    $global_filter$filter_string" /etc/lvm/lvm.conf
+    sudo sed -i "/# global_filter = \[.*\]/a\        $filter_string" /etc/lvm/lvm.conf
    echo_summary "set lvm.conf device global_filter to: $filter_string"
 }

--- a/Show More
+++ b/Show More