Merge "Replace deprecated datetime.utcnow()"

Merge "Globally skip devstack job for pre-commit config update"
Merge "Add image format enforcement toggle"
2024-10-25 08:38:40 +00:00 · 2024-10-25 07:56:22 +00:00 · 2024-10-25 07:56:19 +00:00 · 2024-10-24 00:54:28 +09:00 · 2024-10-21 14:22:36 +00:00 · 2024-10-19 12:55:43 +00:00
139 changed files with 3874 additions and 3072 deletions
@@ -38,3 +38,5 @@ stack-screenrc
 userrc_early
 AUTHORS
 ChangeLog
+tools/dbcounter/build/
+tools/dbcounter/dbcounter.egg-info/
@@ -1,16 +1,18 @@
- pragma:
-   # NOTE(gtema): this is required for the changes in SDK feature/r1 branch to
-   # be using devstack
-   # TODO(gtema): delete this once r1 branch is merged into master
-    implied-branches:
-      - master
-      - feature/r1
-
 - nodeset:
-    name: openstack-single-node
+    name: openstack-single-node-jammy
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: ubuntu-jammy
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: openstack-single-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
    groups:
      - name: tempest
        nodes:
@@ -36,36 +38,6 @@
        nodes:
          - controller

- nodeset:
-    name: openstack-single-node-xenial
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-7
-    nodes:
-      - name: controller
-        label: centos-7
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-8-stream
-    nodes:
-      - name: controller
-        label: centos-8-stream
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
 - nodeset:
    name: devstack-single-node-centos-9-stream
    nodes:
@@ -87,15 +59,18 @@
          - controller

 - nodeset:
-    name: devstack-single-node-fedora-latest
+    name: devstack-single-node-debian-bookworm
    nodes:
      - name: controller
-        label: fedora-35
+        label: debian-bookworm
    groups:
      - name: tempest
        nodes:
          - controller

+# Note(sean-k-mooney): this is still used by horizon for
+# horizon-integration-tests, horizon-integration-pytest and
+# horizon-ui-pytest, remove when horizon is updated.
 - nodeset:
    name: devstack-single-node-debian-bullseye
    nodes:
@@ -107,22 +82,22 @@
          - controller

 - nodeset:
-    name: devstack-single-node-openeuler-20.03-sp2
+    name: devstack-single-node-rockylinux-9
    nodes:
      - name: controller
-        label: openEuler-20-03-LTS-SP2
+        label: rockylinux-9
    groups:
      - name: tempest
        nodes:
          - controller

 - nodeset:
-    name: openstack-two-node
+    name: openstack-two-node-centos-9-stream
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: centos-9-stream
      - name: compute1
-        label: ubuntu-xenial
+        label: centos-9-stream
    groups:
      # Node where tests are executed and test results collected
      - name: tempest
@@ -147,12 +122,42 @@
          - compute1

 - nodeset:
-    name: openstack-two-node-centos-8-stream
+    name: openstack-two-node-jammy
    nodes:
      - name: controller
-        label: centos-8-stream
+        label: ubuntu-jammy
      - name: compute1
-        label: centos-8-stream
+        label: ubuntu-jammy
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
+- nodeset:
+    name: openstack-two-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
+      - name: compute1
+        label: ubuntu-noble
    groups:
      # Node where tests are executed and test results collected
      - name: tempest
@@ -236,36 +241,6 @@
        nodes:
          - compute1

- nodeset:
-    name: openstack-two-node-xenial
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-      - name: compute1
-        label: ubuntu-xenial
-    groups:
-      # Node where tests are executed and test results collected
-      - name: tempest
-        nodes:
-          - controller
-      # Nodes running the compute service
-      - name: compute
-        nodes:
-          - controller
-          - compute1
-      # Nodes that are not the controller
-      - name: subnode
-        nodes:
-          - compute1
-      # Switch node for multinode networking setup
-      - name: switch
-        nodes:
-          - controller
-      # Peer nodes for multinode networking setup
-      - name: peers
-        nodes:
-          - compute1
-
 - nodeset:
    name: openstack-three-node-focal
    nodes:
@@ -338,7 +313,7 @@

 - job:
    name: devstack-base
-    parent: multinode
+    parent: openstack-multinode-fips
    abstract: true
    description: |
      Base abstract Devstack job.
@@ -353,7 +328,6 @@
    required-projects:
      - opendev.org/openstack/devstack
    roles:
-      - zuul: opendev.org/openstack/devstack-gate
      - zuul: opendev.org/openstack/openstack-zuul-jobs
    vars:
      devstack_localrc:
@@ -387,8 +361,10 @@
        '{{ devstack_log_dir }}/devstacklog.txt.summary': logs
        '{{ devstack_log_dir }}/tcpdump.pcap': logs
        '{{ devstack_log_dir }}/worlddump-latest.txt': logs
+        '{{ devstack_log_dir }}/qemu.coredump': logs
        '{{ devstack_full_log}}': logs
        '{{ stage_dir }}/verify_tempest_conf.log': logs
+        '{{ stage_dir }}/performance.json': logs
        '{{ stage_dir }}/apache': logs
        '{{ stage_dir }}/apache_config': logs
        '{{ stage_dir }}/etc': logs
@@ -397,6 +373,7 @@
        /var/log/mysql: logs
        /var/log/libvirt: logs
        /etc/libvirt: logs
+        /etc/lvm: logs
        /etc/sudoers: logs
        /etc/sudoers.d: logs
        '{{ stage_dir }}/iptables.txt': logs
@@ -407,6 +384,7 @@
        '{{ stage_dir }}/rpm-qa.txt': logs
        '{{ stage_dir }}/core': logs
        '{{ stage_dir }}/listen53.txt': logs
+        '{{ stage_dir }}/services.txt': logs
        '{{ stage_dir }}/deprecations.log': logs
        '{{ stage_dir }}/audit.log': logs
        /etc/ceph: logs
@@ -454,6 +432,8 @@
      - ^releasenotes/.*$
      # Translations
      - ^.*/locale/.*po$
+      # pre-commit config
+      - ^.pre-commit-config.yaml$

 - job:
    name: devstack-minimal
@@ -461,7 +441,7 @@
    description: |
      Minimal devstack base job, intended for use by jobs that need
      less than the normal minimum set of required-projects.
-    nodeset: openstack-single-node-focal
+    nodeset: openstack-single-node-jammy
    required-projects:
      - opendev.org/openstack/requirements
    vars:
@@ -475,14 +455,18 @@
        dstat: false
        etcd3: true
        memory_tracker: true
+        file_tracker: true
        mysql: true
        rabbit: true
+        openstack-cli-server: true
    group-vars:
      subnode:
        devstack_services:
          # Shared services
          dstat: false
          memory_tracker: true
+          file_tracker: true
+          openstack-cli-server: true
        devstack_localrc:
          # Multinode specific settings
          HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}"
@@ -528,8 +512,17 @@
      - opendev.org/openstack/nova
      - opendev.org/openstack/placement
      - opendev.org/openstack/swift
+      - opendev.org/openstack/os-test-images
    timeout: 7200
    vars:
+      # based on observation of the integrated gate
+      # tempest-integrated-compute was only using ~1.7GB of swap
+      # when zswap and the host turning are enabled that increase
+      # slightly to ~2GB. we are setting the swap size to 8GB to
+      # be safe and account for more complex scenarios.
+      # we should revisit this value after some time to see if we
+      # can reduce it.
+      configure_swap_size: 8192
      devstack_localrc:
        # Common OpenStack services settings
        SWIFT_REPLICAS: 1
@@ -538,6 +531,26 @@
        DEBUG_LIBVIRT_COREDUMPS: true
        NOVA_VNC_ENABLED: true
        OVN_DBS_LOG_LEVEL: dbg
+        # tune the host to optimize memory usage and hide io latency
+        # these setting will configure the kernel to treat the host page
+        # cache and swap with equal priority, and prefer deferring writes
+        # changing the default swappiness, dirty_ratio and
+        # the vfs_cache_pressure
+        ENABLE_SYSCTL_MEM_TUNING: true
+        # the net tuning optimizes ipv4 tcp fast open and config the default
+        # qdisk policy to pfifo_fast which effectively disable all qos.
+        # this minimizes the cpu load of the host network stack
+        ENABLE_SYSCTL_NET_TUNING: true
+        # zswap allows the kernel to compress pages in memory before swapping
+        # them to disk. this can reduce the amount of swap used and improve
+        # performance. effectively this trades a small amount of cpu for an
+        # increase in swap performance by reducing the amount of data
+        # written to disk. the overall speedup is proportional to the
+        # compression ratio and the speed of the swap device.
+        # NOTE: this option is ignored when not using nova with the libvirt
+        # virt driver.
+        NOVA_LIBVIRT_TB_CACHE_SIZE: 128
+        ENABLE_ZSWAP: true
      devstack_local_conf:
        post-config:
          $NEUTRON_CONF:
@@ -550,6 +563,7 @@
        dstat: false
        etcd3: true
        memory_tracker: true
+        file_tracker: true
        mysql: true
        rabbit: true
        tls-proxy: true
@@ -599,6 +613,7 @@
          # Shared services
          dstat: false
          memory_tracker: true
+          file_tracker: true
          tls-proxy: true
          # Nova services
          n-cpu: true
@@ -628,16 +643,38 @@
          Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
          NOVA_VNC_ENABLED: true
          ENABLE_CHASSIS_AS_GW: false
+          # tune the host to optimize memory usage and hide io latency
+          # these setting will configure the kernel to treat the host page
+          # cache and swap with equal priority, and prefer deferring writes
+          # changing the default swappiness, dirty_ratio and
+          # the vfs_cache_pressure
+          ENABLE_SYSCTL_MEM_TUNING: true
+          # the net tuning optimizes ipv4 tcp fast open and config the default
+          # qdisk policy to pfifo_fast which effectively disable all qos.
+          # this minimizes the cpu load of the host network stack
+          ENABLE_SYSCTL_NET_TUNING: true
+          # zswap allows the kernel to compress pages in memory before swapping
+          # them to disk. this can reduce the amount of swap used and improve
+          # performance. effectivly this trades a small amount of cpu for an
+          # increase in swap performance by reducing the amount of data
+          # written to disk. the overall speedup is porportional to the
+          # compression ratio and the speed of the swap device.
+          ENABLE_ZSWAP: true
+          # NOTE: this option is ignored when not using nova with the libvirt
+          # virt driver.
+          NOVA_LIBVIRT_TB_CACHE_SIZE: 128

 - job:
    name: devstack-ipv6
    parent: devstack
    description: |
-      Devstack single node job for integration gate with IPv6.
+      Devstack single node job for integration gate with IPv6,
+      all services and tunnels using IPv6 addresses.
    vars:
      devstack_localrc:
        SERVICE_IP_VERSION: 6
        SERVICE_HOST: ""
+        TUNNEL_IP_VERSION: 6

 - job:
    name: devstack-enforce-scope
@@ -646,15 +683,12 @@
      This job runs the devstack with scope checks enabled.
    vars:
      devstack_localrc:
-        # Keep enabeling the services here to run with system scope
-        CINDER_ENFORCE_SCOPE: true
-        GLANCE_ENFORCE_SCOPE: true
-        NEUTRON_ENFORCE_SCOPE: true
+        ENFORCE_SCOPE: true

 - job:
    name: devstack-multinode
    parent: devstack
-    nodeset: openstack-two-node-focal
+    nodeset: openstack-two-node-jammy
    description: |
      Simple multinode test to verify multinode functionality on devstack side.
      This is not meant to be used as a parent job.
@@ -663,36 +697,67 @@
 # we often have to rush things through devstack to stabilise the gate,
 # and these platforms don't have the round-the-clock support to avoid
 # becoming blockers in that situation.
- job:
-    name: devstack-platform-centos-8-stream
-    parent: tempest-full-py3
-    description: CentOS 8 Stream platform test
-    nodeset: devstack-single-node-centos-8-stream
-    voting: false
-    timeout: 9000
-    vars:
-      configure_swap_size: 4096
-
 - job:
    name: devstack-platform-centos-9-stream
    parent: tempest-full-py3
    description: CentOS 9 Stream platform test
    nodeset: devstack-single-node-centos-9-stream
    timeout: 9000
+    voting: false
+
+- job:
+    name: devstack-platform-debian-bookworm
+    parent: tempest-full-py3
+    description: Debian Bookworm platform test
+    nodeset: devstack-single-node-debian-bookworm
+    timeout: 9000
    vars:
      configure_swap_size: 4096

 - job:
-    name: devstack-platform-debian-bullseye
+    name: devstack-platform-rocky-blue-onyx
    parent: tempest-full-py3
-    description: Debian Bullseye platform test
-    nodeset: devstack-single-node-debian-bullseye
+    description: Rocky Linux 9 Blue Onyx platform test
+    nodeset: devstack-single-node-rockylinux-9
+    timeout: 9000
+    # NOTE(danms): This has been failing lately with some repository metadata
+    # errors. We're marking this as non-voting until it appears to have
+    # stabilized:
+    # https://zuul.openstack.org/builds?job_name=devstack-platform-rocky-blue-onyx&skip=0
+    voting: false
+    vars:
+      configure_swap_size: 4096
+
+- job:
+    name: devstack-platform-ubuntu-noble
+    parent: tempest-full-py3
+    description: Ubuntu 24.04 LTS (noble) platform test
+    nodeset: openstack-single-node-noble
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovn-source
+    parent: devstack-platform-ubuntu-jammy
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVN from source)
+    voting: false
+    vars:
+      devstack_localrc:
+        OVN_BUILD_FROM_SOURCE: True
+        OVN_BRANCH: "v21.06.0"
+        OVS_BRANCH: "a4b04276ab5934d087669ff2d191a23931335c87"
+        OVS_SYSCONFDIR: "/usr/local/etc/openvswitch"
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovs
+    parent: tempest-full-py3
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVS)
+    nodeset: openstack-single-node-jammy
    voting: false
    timeout: 9000
    vars:
-      configure_swap_size: 4096
-      # NOTE(yoctozepto): Debian Bullseye does not yet offer OVN. Switch to OVS
-      # for the time being.
+      configure_swap_size: 8192
      devstack_localrc:
        Q_AGENT: openvswitch
        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
@@ -723,20 +788,6 @@
          # Enable Neutron ML2/OVS services
          q-agt: true

- job:
-    name: devstack-platform-openEuler-20.03-SP2
-    parent: tempest-full-py3
-    description: openEuler 20.03 SP2 platform test
-    nodeset: devstack-single-node-openeuler-20.03-sp2
-    voting: false
-    timeout: 9000
-    vars:
-      configure_swap_size: 4096
-      devstack_localrc:
-        # NOTE(wxy):  OVN package is not supported by openEuler yet. Build it
-        # from source instead.
-        OVN_BUILD_FROM_SOURCE: True
-
 - job:
    name: devstack-no-tls-proxy
    parent: tempest-full-py3
@@ -748,30 +799,6 @@
      devstack_services:
        tls-proxy: false

- job:
-    name: devstack-platform-fedora-latest
-    parent: tempest-full-py3
-    description: Fedora latest platform test
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-    vars:
-      configure_swap_size: 4096
-      # Python 3.10 dependency issues; see
-      # https://bugs.launchpad.net/horizon/+bug/1960204
-      devstack_services:
-        horizon: false
-
- job:
-    name: devstack-platform-fedora-latest-virt-preview
-    parent: tempest-full-py3
-    description: Fedora latest platform test using the virt-preview repo.
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-    vars:
-      configure_swap_size: 4096
-      devstack_localrc:
-        ENABLE_FEDORA_VIRT_PREVIEW_REPO: true
-
 - job:
    name: devstack-tox-base
    parent: devstack
@@ -828,7 +855,7 @@

 - job:
    name: devstack-unit-tests
-    nodeset: ubuntu-focal
+    nodeset: ubuntu-jammy
    description: |
      Runs unit tests on devstack project.

@@ -845,11 +872,12 @@
        - devstack
        - devstack-ipv6
        - devstack-enforce-scope
-        - devstack-platform-fedora-latest
-        - devstack-platform-centos-8-stream
        - devstack-platform-centos-9-stream
-        - devstack-platform-debian-bullseye
-        - devstack-platform-openEuler-20.03-SP2
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
        - devstack-multinode
        - devstack-unit-tests
        - openstack-tox-bashate
@@ -863,10 +891,6 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-linuxbridge-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - neutron-ovn-tempest-ovs-release:
            voting: false
            irrelevant-files:
@@ -893,7 +917,11 @@
      jobs:
        - devstack
        - devstack-ipv6
-        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-ubuntu-noble
+        # NOTE(danms): Disabled due to instability, see comment in the job
+        # definition above.
+        # - devstack-platform-rocky-blue-onyx
        - devstack-enforce-scope
        - devstack-multinode
        - devstack-unit-tests
@@ -902,10 +930,6 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-linuxbridge-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - ironic-tempest-bios-ipmi-direct-tinyipa
        - swift-dsvm-functional
        - grenade:
@@ -929,7 +953,9 @@
    # pruned.
    #
    # * nova-next: maintained by nova for unreleased/undefaulted
-    #    things
+    #    things, this job is not experimental but often is used to test
+    #    things that are not yet production ready or to test what will be
+    #    the new default after a deprecation period has ended.
    # * neutron-fullstack-with-uwsgi: maintained by neutron for fullstack test
    #    when neutron-api is served by uwsgi, it's in exprimental for testing.
    #    the next cycle we can remove this  job if things turn out to be
@@ -937,14 +963,10 @@
    # * neutron-functional-with-uwsgi: maintained by neutron for functional
    #    test. Next cycle we can remove this one if things turn out to be
    #    stable engouh with uwsgi.
-    # * neutron-tempest-with-uwsgi: maintained by neutron for tempest test.
+    # * neutron-ovn-tempest-with-uwsgi: maintained by neutron for tempest test.
    #    Next cycle we can remove this if everything run out stable enough.
-    # * nova-multi-cell: maintained by nova and currently non-voting in the
+    # * nova-multi-cell: maintained by nova and now is voting in the
    #    check queue for nova changes but relies on devstack configuration
-    # * devstack-platform-fedora-latest-virt-preview: Maintained by lyarwood
-    #    for Nova to allow early testing of the latest versions of Libvirt and
-    #    QEMU. Should only graduate out of experimental if it ever moves into
-    #    the check queue for Nova.

    experimental:
      jobs:
@@ -952,7 +974,7 @@
        - nova-next
        - neutron-fullstack-with-uwsgi
        - neutron-functional-with-uwsgi
-        - neutron-tempest-with-uwsgi
+        - neutron-ovn-tempest-with-uwsgi
        - devstack-plugin-ceph-tempest-py3:
            irrelevant-files:
              - ^.*\.rst$
@@ -973,8 +995,15 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - devstack-platform-fedora-latest-virt-preview
        - devstack-no-tls-proxy
    periodic:
      jobs:
        - devstack-no-tls-proxy
+    periodic-weekly:
+      jobs:
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
@@ -4,7 +4,7 @@ from git source trees.
 Goals
 =====

-* To quickly build dev OpenStack environments in a clean Ubuntu or Fedora
+* To quickly build dev OpenStack environments in a clean Ubuntu or RockyLinux
  environment
 * To describe working configurations of OpenStack (which code branches
  work together?  what do config files look like for those branches?)
@@ -28,9 +28,9 @@ Versions
 The DevStack master branch generally points to trunk versions of OpenStack
 components.  For older, stable versions, look for branches named
 stable/[release] in the DevStack repo.  For example, you can do the
-following to create a Pike OpenStack cloud::
+following to create a Zed OpenStack cloud::

-    git checkout stable/pike
+    git checkout stable/zed
    ./stack.sh

 You can also pick specific OpenStack project releases by setting the appropriate
@@ -55,7 +55,7 @@ When the script finishes executing, you should be able to access OpenStack
 endpoints, like so:

 * Horizon: http://myhost/
-* Keystone: http://myhost/identity/v2.0/
+* Keystone: http://myhost/identity/v3/

 We also provide an environment file that you can use to interact with your
 cloud via CLI::
@@ -50,7 +50,6 @@ source $TOP_DIR/lib/placement
 source $TOP_DIR/lib/cinder
 source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/neutron
-source $TOP_DIR/lib/neutron-legacy

 set -o xtrace

@@ -4,8 +4,4 @@ Pygments
 docutils
 sphinx>=2.0.0,!=2.1.0 # BSD
 openstackdocstheme>=2.2.1 # Apache-2.0
-nwdiag
-blockdiag
-sphinxcontrib-blockdiag
-sphinxcontrib-nwdiag
 zuul-sphinx>=0.2.0
@@ -23,14 +23,14 @@

 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinx.ext.autodoc',
-               'zuul_sphinx',
-               'openstackdocstheme',
-               'sphinxcontrib.blockdiag',
-               'sphinxcontrib.nwdiag' ]
+extensions = [
+    'sphinx.ext.autodoc',
+    'zuul_sphinx',
+    'openstackdocstheme',
+]

 # openstackdocstheme options
-openstackdocs_repo_name = 'openstack-dev/devstack'
+openstackdocs_repo_name = 'openstack/devstack'
 openstackdocs_pdf_link = True
 openstackdocs_bug_project = 'devstack'
 openstackdocs_bug_tag = ''
@@ -181,6 +181,9 @@ values that most often need to be set.
 If the ``*_PASSWORD`` variables are not set here you will be prompted to
 enter values for them by ``stack.sh``.

+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.
+
 The network ranges must not overlap with any networks in use on the
 host. Overlap is not uncommon as RFC-1918 'private' ranges are commonly
 used for both the local networking and Nova's fixed and floating ranges.
@@ -279,7 +282,7 @@ number of days of old log files to keep.

 ::

-    LOGDAYS=1
+    LOGDAYS=2

 Some coloring is used during the DevStack runs to make it easier to
 see what is going on. This can be disabled with::
@@ -521,8 +524,8 @@ behavior:
 can be configured with any valid IPv6 prefix. The default values make
 use of an auto-generated ``IPV6_GLOBAL_ID`` to comply with RFC4193.

-Service Version
-~~~~~~~~~~~~~~~
+Service IP Version
+~~~~~~~~~~~~~~~~~~

 DevStack can enable service operation over either IPv4 or IPv6 by
 setting ``SERVICE_IP_VERSION`` to either ``SERVICE_IP_VERSION=4`` or
@@ -542,6 +545,27 @@ optionally be used to alter the default IPv6 address::

  HOST_IPV6=${some_local_ipv6_address}

+Tunnel IP Version
+~~~~~~~~~~~~~~~~~
+
+DevStack can enable tunnel operation over either IPv4 or IPv6 by
+setting ``TUNNEL_IP_VERSION`` to either ``TUNNEL_IP_VERSION=4`` or
+``TUNNEL_IP_VERSION=6`` respectively.
+
+When set to ``4`` Neutron will use an IPv4 address for tunnel endpoints,
+for example, ``HOST_IP``.
+
+When set to ``6`` Neutron will use an IPv6 address for tunnel endpoints,
+for example, ``HOST_IPV6``.
+
+The default value for this setting is ``4``.  Dual-mode support, for
+example ``4+6`` is not supported, as this value must match the address
+family of the local tunnel endpoint IP(v6) address.
+
+The value of ``TUNNEL_IP_VERSION`` has a direct relationship to the
+setting of ``TUNNEL_ENDPOINT_IP``, which will default to ``HOST_IP``
+when set to ``4``, and ``HOST_IPV6`` when set to ``6``.
+
 Multi-node setup
 ~~~~~~~~~~~~~~~~

@@ -615,7 +639,7 @@ tests can be run as follows:
 ::

    $ cd /opt/stack/tempest
-    $ tox -efull  tempest.scenario.test_network_basic_ops
+    $ tox -e smoke

 By default tempest is downloaded and the config file is generated, but the
 tempest package is not installed in the system's global site-packages (the
@@ -648,6 +672,35 @@ adjusted by setting ``CINDER_QUOTA_VOLUMES``, ``CINDER_QUOTA_BACKUPS``,
 or ``CINDER_QUOTA_SNAPSHOTS`` to the desired value.  (The default for
 each is 10.)

+DevStack's Cinder LVM configuration module currently supports both iSCSI and
+NVMe connections, and we can choose which one to use with options
+``CINDER_TARGET_HELPER``, ``CINDER_TARGET_PROTOCOL``, ``CINDER_TARGET_PREFIX``,
+and ``CINDER_TARGET_PORT``.
+
+Defaults use iSCSI with the LIO target manager::
+
+  CINDER_TARGET_HELPER="lioadm"
+  CINDER_TARGET_PROTOCOL="iscsi"
+  CINDER_TARGET_PREFIX="iqn.2010-10.org.openstack:"
+  CINDER_TARGET_PORT=3260
+
+Additionally there are 3 supported transport protocols for NVMe,
+``nvmet_rdma``, ``nvmet_tcp``, and ``nvmet_fc``, and when the ``nvmet`` target
+is selected the protocol, prefix, and port defaults will change to more
+sensible defaults for NVMe::
+
+  CINDER_TARGET_HELPER="nvmet"
+  CINDER_TARGET_PROTOCOL="nvmet_rdma"
+  CINDER_TARGET_PREFIX="nvme-subsystem-1"
+  CINDER_TARGET_PORT=4420
+
+When selecting the RDMA transport protocol DevStack will create on Cinder nodes
+a Software RoCE device on top of the ``HOST_IP_IFACE`` and if it is not defined
+then on top of the interface with IP address ``HOST_IP`` or ``HOST_IPV6``.
+
+This Soft-RoCE device will always be created on the Nova compute side since we
+cannot tell beforehand whether there will be an RDMA connection or not.
+

 Keystone
 ~~~~~~~~
@@ -698,7 +751,7 @@ or at runtime via:

 ::

-    openstack --os-cloud devstack-system-admin registered limit update \
+    openstack --os-cloud devstack-system-admin registered limit set \
      --service glance --default-limit 5000 --region RegionOne image_size_total

 .. _arch-configuration:
@@ -42,8 +42,9 @@ Getting Your Patch Merged
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 All changes proposed to the Devstack require two ``Code-Review +2`` votes from
 Devstack core reviewers before one of the core reviewers can approve the patch
-by giving ``Workflow +1`` vote. One exception is for patches to unblock the gate
-which can be approved by single core reviewers.
+by giving ``Workflow +1`` vote. There are 2 exceptions, approving patches to
+unblock the gate and patches that do not relate to the Devstack's core logic,
+like for example old job cleanups, can be approved by single core reviewers.

 Project Team Lead Duties
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -20,6 +20,12 @@ provides consumption output when available memory is seen to be
 falling (i.e. processes are consuming memory).  It also provides
 output showing locked (unswappable) memory.

+file_tracker
+------------
+
+The ``file_tracker`` service periodically monitors the number of
+open files in the system.
+
 tcpdump
 -------

@@ -20,7 +20,7 @@ Walk through various setups used by stackers
   guides/neutron
   guides/devstack-with-nested-kvm
   guides/nova
-   guides/devstack-with-lbaas-v2
+   guides/devstack-with-octavia
   guides/devstack-with-ldap

 All-In-One Single VM
@@ -69,10 +69,10 @@ Nova and devstack

 Guide to working with nova features :doc:`Nova and devstack <guides/nova>`.

-Configure Load-Balancer Version 2
-----------------------------------
+Configure Octavia
+-----------------

-Guide on :doc:`Configure Load-Balancer Version 2 <guides/devstack-with-lbaas-v2>`.
+Guide on :doc:`Configure Octavia <guides/devstack-with-octavia>`.

 Deploying DevStack with LDAP
 ----------------------------
@@ -1,145 +0,0 @@
-Devstack with Octavia Load Balancing
-====================================
-
-Starting with the OpenStack Pike release, Octavia is now a standalone service
-providing load balancing services for OpenStack.
-
-This guide will show you how to create a devstack with `Octavia API`_ enabled.
-
-.. _Octavia API: https://docs.openstack.org/api-ref/load-balancer/v2/index.html
-
-Phase 1: Create DevStack + 2 nova instances
--------------------------------------------
-
-First, set up a vm of your choice with at least 8 GB RAM and 16 GB disk space,
-make sure it is updated. Install git and any other developer tools you find
-useful.
-
-Install devstack
-
-::
-
-    git clone https://opendev.org/openstack/devstack
-    cd devstack/tools
-    sudo ./create-stack-user.sh
-    cd ../..
-    sudo mv devstack /opt/stack
-    sudo chown -R stack.stack /opt/stack/devstack
-
-This will clone the current devstack code locally, then setup the "stack"
-account that devstack services will run under. Finally, it will move devstack
-into its default location in /opt/stack/devstack.
-
-Edit your ``/opt/stack/devstack/local.conf`` to look like
-
-::
-
-    [[local|localrc]]
-    enable_plugin octavia https://opendev.org/openstack/octavia
-    # If you are enabling horizon, include the octavia dashboard
-    # enable_plugin octavia-dashboard https://opendev.org/openstack/octavia-dashboard.git
-    # If you are enabling barbican for TLS offload in Octavia, include it here.
-    # enable_plugin barbican https://opendev.org/openstack/barbican
-
-    # ===== BEGIN localrc =====
-    DATABASE_PASSWORD=password
-    ADMIN_PASSWORD=password
-    SERVICE_PASSWORD=password
-    SERVICE_TOKEN=password
-    RABBIT_PASSWORD=password
-    # Enable Logging
-    LOGFILE=$DEST/logs/stack.sh.log
-    VERBOSE=True
-    LOG_COLOR=True
-    # Pre-requisite
-    ENABLED_SERVICES=rabbit,mysql,key
-    # Horizon - enable for the OpenStack web GUI
-    # ENABLED_SERVICES+=,horizon
-    # Nova
-    ENABLED_SERVICES+=,n-api,n-crt,n-cpu,n-cond,n-sch,n-api-meta,n-sproxy
-    ENABLED_SERVICES+=,placement-api,placement-client
-    # Glance
-    ENABLED_SERVICES+=,g-api
-    # Neutron
-    ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron
-    ENABLED_SERVICES+=,octavia,o-cw,o-hk,o-hm,o-api
-    # Cinder
-    ENABLED_SERVICES+=,c-api,c-vol,c-sch
-    # Tempest
-    ENABLED_SERVICES+=,tempest
-    # Barbican - Optionally used for TLS offload in Octavia
-    # ENABLED_SERVICES+=,barbican
-    # ===== END localrc =====
-
-Run stack.sh and do some sanity checks
-
-::
-
-    sudo su - stack
-    cd /opt/stack/devstack
-    ./stack.sh
-    . ./openrc
-
-    openstack network list  # should show public and private networks
-
-Create two nova instances that we can use as test http servers:
-
-::
-
-    #create nova instances on private network
-    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node1
-    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node2
-    openstack server list # should show the nova instances just created
-
-    #add secgroup rules to allow ssh etc..
-    openstack security group rule create default --protocol icmp
-    openstack security group rule create default --protocol tcp --dst-port 22:22
-    openstack security group rule create default --protocol tcp --dst-port 80:80
-
-Set up a simple web server on each of these instances. ssh into each instance (username 'cirros', password 'cubswin:)' or 'gocubsgo') and run
-
-::
-
-    MYIP=$(ifconfig eth0|grep 'inet addr'|awk -F: '{print $2}'| awk '{print $1}')
-    while true; do echo -e "HTTP/1.0 200 OK\r\n\r\nWelcome to $MYIP" | sudo nc -l -p 80 ; done&
-
-Phase 2: Create your load balancer
----------------------------------
-
-Make sure you have the 'openstack loadbalancer' commands:
-
-::
-
-    pip install python-octaviaclient
-
-Create your load balancer:
-
-::
-
-    openstack loadbalancer create --name lb1 --vip-subnet-id private-subnet
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer listener create --protocol HTTP --protocol-port 80 --name listener1 lb1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer pool create --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer healthmonitor create --delay 5 --timeout 2 --max-retries 1 --type HTTP pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer member create --subnet-id private-subnet --address <web server 1 address> --protocol-port 80 pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer member create --subnet-id private-subnet --address <web server 2 address> --protocol-port 80 pool1
-
-Please note: The <web server # address> fields are the IP addresses of the nova
-servers created in Phase 1.
-Also note, using the API directly you can do all of the above commands in one
-API call.
-
-Phase 3: Test your load balancer
--------------------------------
-
-::
-
-    openstack loadbalancer show lb1 # Note the vip_address
-    curl http://<vip_address>
-    curl http://<vip_address>
-
-This should show the "Welcome to <IP>" message from each member server.
@@ -1,3 +1,5 @@
+.. _kvm_nested_virt:
+
 =======================================================
 Configure DevStack with KVM-based Nested Virtualization
 =======================================================
@@ -0,0 +1,144 @@
+Devstack with Octavia Load Balancing
+====================================
+
+Starting with the OpenStack Pike release, Octavia is now a standalone service
+providing load balancing services for OpenStack.
+
+This guide will show you how to create a devstack with `Octavia API`_ enabled.
+
+.. _Octavia API: https://docs.openstack.org/api-ref/load-balancer/v2/index.html
+
+Phase 1: Create DevStack + 2 nova instances
+--------------------------------------------
+
+First, set up a VM of your choice with at least 8 GB RAM and 16 GB disk space,
+make sure it is updated. Install git and any other developer tools you find
+useful.
+
+Install devstack::
+
+    git clone https://opendev.org/openstack/devstack
+    cd devstack/tools
+    sudo ./create-stack-user.sh
+    cd ../..
+    sudo mv devstack /opt/stack
+    sudo chown -R stack.stack /opt/stack/devstack
+
+This will clone the current devstack code locally, then setup the "stack"
+account that devstack services will run under. Finally, it will move devstack
+into its default location in /opt/stack/devstack.
+
+Edit your ``/opt/stack/devstack/local.conf`` to look like::
+
+    [[local|localrc]]
+    # ===== BEGIN localrc =====
+    DATABASE_PASSWORD=password
+    ADMIN_PASSWORD=password
+    SERVICE_PASSWORD=password
+    SERVICE_TOKEN=password
+    RABBIT_PASSWORD=password
+    GIT_BASE=https://opendev.org
+    # Optional settings:
+    # OCTAVIA_AMP_BASE_OS=centos
+    # OCTAVIA_AMP_DISTRIBUTION_RELEASE_ID=9-stream
+    # OCTAVIA_AMP_IMAGE_SIZE=3
+    # OCTAVIA_LB_TOPOLOGY=ACTIVE_STANDBY
+    # OCTAVIA_ENABLE_AMPHORAV2_JOBBOARD=True
+    # LIBS_FROM_GIT+=octavia-lib,
+    # Enable Logging
+    LOGFILE=$DEST/logs/stack.sh.log
+    VERBOSE=True
+    LOG_COLOR=True
+    enable_service rabbit
+    enable_plugin neutron $GIT_BASE/openstack/neutron
+    # Octavia supports using QoS policies on the VIP port:
+    enable_service q-qos
+    enable_service placement-api placement-client
+    # Octavia services
+    enable_plugin octavia $GIT_BASE/openstack/octavia master
+    enable_plugin octavia-dashboard $GIT_BASE/openstack/octavia-dashboard
+    enable_plugin ovn-octavia-provider $GIT_BASE/openstack/ovn-octavia-provider
+    enable_plugin octavia-tempest-plugin $GIT_BASE/openstack/octavia-tempest-plugin
+    enable_service octavia o-api o-cw o-hm o-hk o-da
+    # If you are enabling barbican for TLS offload in Octavia, include it here.
+    # enable_plugin barbican $GIT_BASE/openstack/barbican
+    # enable_service barbican
+    # Cinder (optional)
+    disable_service c-api c-vol c-sch
+    # Tempest
+    enable_service tempest
+    # ===== END localrc =====
+
+.. note::
+    For best performance it is highly recommended to use KVM
+    virtualization instead of QEMU.
+    Also make sure nested virtualization is enabled as documented in
+    :ref:`the respective guide <kvm_nested_virt>`.
+    By adding ``LIBVIRT_CPU_MODE="host-passthrough"`` to your
+    ``local.conf`` you enable the guest VMs to make use of all features your
+    host's CPU provides.
+
+Run stack.sh and do some sanity checks::
+
+    sudo su - stack
+    cd /opt/stack/devstack
+    ./stack.sh
+    . ./openrc
+
+    openstack network list  # should show public and private networks
+
+Create two nova instances that we can use as test http servers::
+
+    # create nova instances on private network
+    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node1
+    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node2
+    openstack server list # should show the nova instances just created
+
+    # add secgroup rules to allow ssh etc..
+    openstack security group rule create default --protocol icmp
+    openstack security group rule create default --protocol tcp --dst-port 22:22
+    openstack security group rule create default --protocol tcp --dst-port 80:80
+
+Set up a simple web server on each of these instances. One possibility is to use
+the `Golang test server`_ that is used by the Octavia project for CI testing
+as well.
+Copy the binary to your instances and start it as shown below
+(username 'cirros', password 'gocubsgo')::
+
+    INST_IP=<instance IP>
+    scp -O test_server.bin cirros@${INST_IP}:
+    ssh -f cirros@${INST_IP} ./test_server.bin -id ${INST_IP}
+
+When started this way the test server will respond to HTTP requests with
+its own IP.
+
+Phase 2: Create your load balancer
+----------------------------------
+
+Create your load balancer::
+
+    openstack loadbalancer create --wait --name lb1 --vip-subnet-id private-subnet
+    openstack loadbalancer listener create --wait --protocol HTTP --protocol-port 80 --name listener1 lb1
+    openstack loadbalancer pool create --wait --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
+    openstack loadbalancer healthmonitor create --wait --delay 5 --timeout 2 --max-retries 1 --type HTTP pool1
+    openstack loadbalancer member create --wait --subnet-id private-subnet --address <web server 1 address> --protocol-port 80 pool1
+    openstack loadbalancer member create --wait --subnet-id private-subnet --address <web server 2 address> --protocol-port 80 pool1
+
+Please note: The <web server # address> fields are the IP addresses of the nova
+servers created in Phase 1.
+Also note, using the API directly you can do all of the above commands in one
+API call.
+
+Phase 3: Test your load balancer
+--------------------------------
+
+::
+
+    openstack loadbalancer show lb1 # Note the vip_address
+    curl http://<vip_address>
+    curl http://<vip_address>
+
+This should show the "Welcome to <IP>" message from each member server.
+
+
+.. _Golang test server: https://opendev.org/openstack/octavia-tempest-plugin/src/branch/master/octavia_tempest_plugin/contrib/test_server
@@ -75,13 +75,21 @@ Otherwise create the stack user:

    useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+::
+
+    chmod +x /opt/stack
+
 This user will be making many changes to your system during installation
 and operation so it needs to have sudo privileges to root without a
 password:

 ::

-    echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 From here on use the ``stack`` user. **Logout** and **login** as the
 ``stack`` user.
@@ -41,19 +41,8 @@ network and is on a shared subnet with other machines.  The
 `local.conf` exhibited here assumes that 1500 is a reasonable MTU to
 use on that network.

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                }
-        }
+.. image:: /assets/images/neutron-network-1.png
+   :alt: Network configuration for a single DevStack node


 DevStack Configuration
@@ -100,21 +89,8 @@ also want to do multinode testing and networking.
 Physical Network Setup
 ~~~~~~~~~~~~~~~~~~~~~~

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                        devstack-2 [ address = "172.18.161.7" ];
-                }
-        }
-
+.. image:: /assets/images/neutron-network-2.png
+   :alt: Network configuration for multiple DevStack nodes

 After DevStack installs and configures Neutron, traffic from guest VMs
 flows out of `devstack-2` (the compute node) and is encapsulated in a
@@ -222,8 +198,6 @@ connect OpenStack nodes (like `devstack-2`) together. This bridge is
 used so that project network traffic, using the VXLAN tunneling
 protocol, flows between each compute node where project instances run.

-
-
 DevStack Compute Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -268,30 +242,8 @@ to the neutron L3 service.
 Physical Network Setup
 ----------------------

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network provider_net {
-                        address = "203.0.113.0/24"
-                        router [ address = "203.0.113.1" ];
-                        controller;
-                        compute1;
-                        compute2;
-                }
-
-                network control_plane {
-                        router [ address = "10.0.0.1" ]
-                        address = "10.0.0.0/24"
-                        controller [ address = "10.0.0.2" ]
-                        compute1 [ address = "10.0.0.3" ]
-                        compute2 [ address = "10.0.0.4" ]
-                }
-        }
-
+.. image:: /assets/images/neutron-network-3.png
+   :alt: Network configuration for provider networks

 On a compute node, the first interface, eth0 is used for the OpenStack
 management (API, message bus, etc) as well as for ssh for an
@@ -122,7 +122,7 @@ when creating the server, for example:
 .. code-block:: shell

  $ openstack --os-compute-api-version 2.37 server create --flavor cirros256 \
-      --image cirros-0.3.5-x86_64-disk --nic none --wait test-server
+      --image cirros-0.6.3-x86_64-disk --nic none --wait test-server

 .. note:: ``--os-compute-api-version`` greater than or equal to 2.37 is
          required to use ``--nic=none``.
@@ -49,13 +49,21 @@ below)

    $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it will need
 to have sudo privileges:

 .. code-block:: console

    $ apt-get install sudo -y || yum install -y sudo
-    $ echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    $ echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 .. note:: On some systems you may need to use ``sudo visudo``.

@@ -98,6 +106,9 @@ do the following:
 -  Set the service password. This is used by the OpenStack services
   (Nova, Glance, etc) to authenticate with Keystone.

+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.
+
 ``local.conf`` should look something like this:

 .. code-block:: ini
@@ -37,10 +37,10 @@ Install Linux
 -------------

 Start with a clean and minimal install of a Linux system. DevStack
-attempts to support the two latest LTS releases of Ubuntu, the
-latest/current Fedora version, CentOS/RHEL 8, OpenSUSE and openEuler.
+attempts to support the two latest LTS releases of Ubuntu,
+Rocky Linux 9 and openEuler.

-If you do not have a preference, Ubuntu 20.04 (Focal Fossa) is the
+If you do not have a preference, Ubuntu 22.04 (Jammy) is the
 most tested, and will probably go the smoothest.

 Add Stack User (optional)
@@ -57,6 +57,14 @@ to run DevStack with

   $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it should
 have sudo privileges:

@@ -93,7 +101,10 @@ devstack git repo.
 This is the minimum required config to get started with DevStack.

 .. note:: There is a sample :download:`local.conf </assets/local.conf>` file
-    under the *samples* directory in the devstack repository.
+   under the *samples* directory in the devstack repository.
+
+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.

 Start the install
 -----------------
@@ -102,7 +113,7 @@ Start the install

   $ ./stack.sh

-This will take a 15 - 20 minutes, largely depending on the speed of
+This will take 15 - 30 minutes, largely depending on the speed of
 your internet connection. Many git trees and packages will be
 installed during this process.

@@ -122,6 +133,8 @@ there.
 You can ``source openrc`` in your shell, and then use the
 ``openstack`` command line tool to manage your devstack.

+You can :ref:`create a VM and SSH into it <ssh>`.
+
 You can ``cd /opt/stack/tempest`` and run tempest tests that have
 been configured to work with your devstack.

@@ -68,7 +68,7 @@ Shared Guest Interface
 .. warning::

   This is not a recommended configuration. Because of interactions
-   between ovs and bridging, if you reboot your box with active
+   between OVS and bridging, if you reboot your box with active
   networking you may lose network connectivity to your system.

 If you need your guests accessible on the network, but only have 1
@@ -114,3 +114,125 @@ For IPv6, ``FIXED_RANGE_V6`` will default to the first /64 of the value of
 ``FIXED_RANGE_V6`` will just use the value of that directly.
 ``SUBNETPOOL_PREFIX_V6`` will just default to the value of
 ``IPV6_ADDRS_SAFE_TO_USE`` directly.
+
+.. _ssh:
+
+SSH access to instances
+=======================
+
+To validate connectivity, you can create an instance using the
+``$PRIVATE_NETWORK_NAME`` network (default: ``private``), create a floating IP
+using the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``), and attach
+this floating IP to the instance:
+
+.. code-block:: shell
+
+    openstack keypair create --public-key ~/.ssh/id_rsa.pub test-keypair
+    openstack server create --network private --key-name test-keypair ... test-server
+    fip_id=$(openstack floating ip create public -f value -c id)
+    openstack server add floating ip test-server ${fip_id}
+
+Once done, ensure you have enabled SSH and ICMP (ping) access for the security
+group used for the instance. You can either create a custom security group and
+specify it when creating the instance or add it after creation, or you can
+modify the ``default`` security group created by default for each project.
+Let's do the latter:
+
+.. code-block:: shell
+
+    openstack security group rule create --proto icmp --dst-port 0 default
+    openstack security group rule create --proto tcp --dst-port 22 default
+
+Finally, SSH into the instance. If you used the Cirros instance uploaded by
+default, then you can run the following:
+
+.. code-block:: shell
+
+    openstack server ssh test-server -- -l cirros
+
+This will connect using the ``cirros`` user and the keypair you configured when
+creating the instance.
+
+Remote SSH access to instances
+==============================
+
+You can also SSH to created instances on your DevStack host from other hosts.
+This can be helpful if you are e.g. deploying DevStack in a VM on an existing
+cloud and wish to do development on your local machine. There are a few ways to
+do this.
+
+.. rubric:: Configure instances to be locally accessible
+
+The most obvious way is to configure guests to be locally accessible, as
+described `above <Locally Accessible Guests>`__. This has the advantage of
+requiring no further effort on the client. However, it is more involved and
+requires either support from your cloud or some inadvisable workarounds.
+
+.. rubric:: Use your DevStack host as a jump host
+
+You can choose to use your DevStack host as a jump host. To SSH to a instance
+this way, pass the standard ``-J`` option to the ``openstack ssh`` / ``ssh``
+command. For example:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -J username@devstack-host
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+This can also be configured via your ``~/.ssh/config`` file, making it rather
+effortless. However, it only allows SSH access. If you want to access e.g. a
+web application on the instance, you will need to configure an SSH tunnel and
+forward select ports using the ``-L`` option. For example, to forward HTTP
+traffic:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -L 8080:username@devstack-host:80
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+As you can imagine, this can quickly get out of hand, particularly for more
+complex guest applications with multiple ports.
+
+.. rubric:: Use a proxy or VPN tool
+
+You can use a proxy or VPN tool to enable tunneling for the floating IP
+address range of the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``)
+defined by ``$FLOATING_RANGE`` (default: ``172.24.4.0/24``). There are many
+such tools available to do this. For example, we could use a useful utility
+called `shuttle`__. To enable tunneling using ``shuttle``, first ensure you
+have allowed SSH and HTTP(S) traffic to your DevStack host. Allowing HTTP(S)
+traffic is necessary so you can use the OpenStack APIs remotely. How you do
+this will depend on where your DevStack host is running. Once this is done,
+install ``sshuttle`` on your localhost:
+
+.. code-block:: bash
+
+    sudo apt-get install sshuttle || yum install sshuttle
+
+Finally, start ``sshuttle`` on your localhost using the floating IP address
+range. For example, assuming you are using the default value for
+``$FLOATING_RANGE``, you can do:
+
+.. code-block:: bash
+
+    sshuttle -r username@devstack-host 172.24.4.0/24
+
+(where ``username`` and ``devstack-host`` are the username and hostname of your
+DevStack host).
+
+You should now be able to create an instance and SSH into it:
+
+.. code-block:: bash
+
+    openstack server ssh test-server -- -l cirros
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`)
+
+.. __: https://github.com/sshuttle/sshuttle
@@ -23,13 +23,12 @@ strategy to include the latest Ubuntu release and the latest RHEL
 release.*

 -  Ubuntu: current LTS release plus current development release
-  Fedora: current release plus previous release
-  RHEL/CentOS: current major release
+-  RHEL/CentOS/RockyLinux: current major release
 -  Other OS platforms may continue to be included but the maintenance of
   those platforms shall not be assumed simply due to their presence.
   Having a listed point-of-contact for each additional OS will greatly
   increase its chance of being well-maintained.
-  Patches for Ubuntu and/or Fedora will not be held up due to
+-  Patches for Ubuntu and/or RockyLinux will not be held up due to
   side-effects on other OS platforms.

 Databases
@@ -28,8 +28,6 @@ openstack/aodh                           `https://opendev.org/openstack/aodh <ht
 openstack/barbican                       `https://opendev.org/openstack/barbican <https://opendev.org/openstack/barbican>`__
 openstack/blazar                         `https://opendev.org/openstack/blazar <https://opendev.org/openstack/blazar>`__
 openstack/ceilometer                     `https://opendev.org/openstack/ceilometer <https://opendev.org/openstack/ceilometer>`__
-openstack/ceilometer-powervm             `https://opendev.org/openstack/ceilometer-powervm <https://opendev.org/openstack/ceilometer-powervm>`__
-openstack/cinderlib                      `https://opendev.org/openstack/cinderlib <https://opendev.org/openstack/cinderlib>`__
 openstack/cloudkitty                     `https://opendev.org/openstack/cloudkitty <https://opendev.org/openstack/cloudkitty>`__
 openstack/cyborg                         `https://opendev.org/openstack/cyborg <https://opendev.org/openstack/cyborg>`__
 openstack/designate                      `https://opendev.org/openstack/designate <https://opendev.org/openstack/designate>`__
@@ -39,7 +37,6 @@ openstack/devstack-plugin-container      `https://opendev.org/openstack/devstack
 openstack/devstack-plugin-kafka          `https://opendev.org/openstack/devstack-plugin-kafka <https://opendev.org/openstack/devstack-plugin-kafka>`__
 openstack/devstack-plugin-nfs            `https://opendev.org/openstack/devstack-plugin-nfs <https://opendev.org/openstack/devstack-plugin-nfs>`__
 openstack/devstack-plugin-open-cas       `https://opendev.org/openstack/devstack-plugin-open-cas <https://opendev.org/openstack/devstack-plugin-open-cas>`__
-openstack/ec2-api                        `https://opendev.org/openstack/ec2-api <https://opendev.org/openstack/ec2-api>`__
 openstack/freezer                        `https://opendev.org/openstack/freezer <https://opendev.org/openstack/freezer>`__
 openstack/freezer-api                    `https://opendev.org/openstack/freezer-api <https://opendev.org/openstack/freezer-api>`__
 openstack/freezer-tempest-plugin         `https://opendev.org/openstack/freezer-tempest-plugin <https://opendev.org/openstack/freezer-tempest-plugin>`__
@@ -51,9 +48,7 @@ openstack/ironic-inspector               `https://opendev.org/openstack/ironic-i
 openstack/ironic-prometheus-exporter     `https://opendev.org/openstack/ironic-prometheus-exporter <https://opendev.org/openstack/ironic-prometheus-exporter>`__
 openstack/ironic-ui                      `https://opendev.org/openstack/ironic-ui <https://opendev.org/openstack/ironic-ui>`__
 openstack/keystone                       `https://opendev.org/openstack/keystone <https://opendev.org/openstack/keystone>`__
-openstack/kuryr-kubernetes               `https://opendev.org/openstack/kuryr-kubernetes <https://opendev.org/openstack/kuryr-kubernetes>`__
 openstack/kuryr-libnetwork               `https://opendev.org/openstack/kuryr-libnetwork <https://opendev.org/openstack/kuryr-libnetwork>`__
-openstack/kuryr-tempest-plugin           `https://opendev.org/openstack/kuryr-tempest-plugin <https://opendev.org/openstack/kuryr-tempest-plugin>`__
 openstack/magnum                         `https://opendev.org/openstack/magnum <https://opendev.org/openstack/magnum>`__
 openstack/magnum-ui                      `https://opendev.org/openstack/magnum-ui <https://opendev.org/openstack/magnum-ui>`__
 openstack/manila                         `https://opendev.org/openstack/manila <https://opendev.org/openstack/manila>`__
@@ -64,14 +59,10 @@ openstack/mistral                        `https://opendev.org/openstack/mistral
 openstack/monasca-api                    `https://opendev.org/openstack/monasca-api <https://opendev.org/openstack/monasca-api>`__
 openstack/monasca-events-api             `https://opendev.org/openstack/monasca-events-api <https://opendev.org/openstack/monasca-events-api>`__
 openstack/monasca-tempest-plugin         `https://opendev.org/openstack/monasca-tempest-plugin <https://opendev.org/openstack/monasca-tempest-plugin>`__
-openstack/murano                         `https://opendev.org/openstack/murano <https://opendev.org/openstack/murano>`__
 openstack/networking-bagpipe             `https://opendev.org/openstack/networking-bagpipe <https://opendev.org/openstack/networking-bagpipe>`__
 openstack/networking-baremetal           `https://opendev.org/openstack/networking-baremetal <https://opendev.org/openstack/networking-baremetal>`__
 openstack/networking-bgpvpn              `https://opendev.org/openstack/networking-bgpvpn <https://opendev.org/openstack/networking-bgpvpn>`__
 openstack/networking-generic-switch      `https://opendev.org/openstack/networking-generic-switch <https://opendev.org/openstack/networking-generic-switch>`__
-openstack/networking-hyperv              `https://opendev.org/openstack/networking-hyperv <https://opendev.org/openstack/networking-hyperv>`__
-openstack/networking-odl                 `https://opendev.org/openstack/networking-odl <https://opendev.org/openstack/networking-odl>`__
-openstack/networking-powervm             `https://opendev.org/openstack/networking-powervm <https://opendev.org/openstack/networking-powervm>`__
 openstack/networking-sfc                 `https://opendev.org/openstack/networking-sfc <https://opendev.org/openstack/networking-sfc>`__
 openstack/neutron                        `https://opendev.org/openstack/neutron <https://opendev.org/openstack/neutron>`__
 openstack/neutron-dynamic-routing        `https://opendev.org/openstack/neutron-dynamic-routing <https://opendev.org/openstack/neutron-dynamic-routing>`__
@@ -80,22 +71,17 @@ openstack/neutron-fwaas-dashboard        `https://opendev.org/openstack/neutron-
 openstack/neutron-tempest-plugin         `https://opendev.org/openstack/neutron-tempest-plugin <https://opendev.org/openstack/neutron-tempest-plugin>`__
 openstack/neutron-vpnaas                 `https://opendev.org/openstack/neutron-vpnaas <https://opendev.org/openstack/neutron-vpnaas>`__
 openstack/neutron-vpnaas-dashboard       `https://opendev.org/openstack/neutron-vpnaas-dashboard <https://opendev.org/openstack/neutron-vpnaas-dashboard>`__
-openstack/nova-powervm                   `https://opendev.org/openstack/nova-powervm <https://opendev.org/openstack/nova-powervm>`__
+openstack/nova                           `https://opendev.org/openstack/nova <https://opendev.org/openstack/nova>`__
 openstack/octavia                        `https://opendev.org/openstack/octavia <https://opendev.org/openstack/octavia>`__
 openstack/octavia-dashboard              `https://opendev.org/openstack/octavia-dashboard <https://opendev.org/openstack/octavia-dashboard>`__
 openstack/octavia-tempest-plugin         `https://opendev.org/openstack/octavia-tempest-plugin <https://opendev.org/openstack/octavia-tempest-plugin>`__
 openstack/openstacksdk                   `https://opendev.org/openstack/openstacksdk <https://opendev.org/openstack/openstacksdk>`__
 openstack/osprofiler                     `https://opendev.org/openstack/osprofiler <https://opendev.org/openstack/osprofiler>`__
-openstack/oswin-tempest-plugin           `https://opendev.org/openstack/oswin-tempest-plugin <https://opendev.org/openstack/oswin-tempest-plugin>`__
+openstack/ovn-bgp-agent                  `https://opendev.org/openstack/ovn-bgp-agent <https://opendev.org/openstack/ovn-bgp-agent>`__
 openstack/ovn-octavia-provider           `https://opendev.org/openstack/ovn-octavia-provider <https://opendev.org/openstack/ovn-octavia-provider>`__
-openstack/patrole                        `https://opendev.org/openstack/patrole <https://opendev.org/openstack/patrole>`__
 openstack/rally-openstack                `https://opendev.org/openstack/rally-openstack <https://opendev.org/openstack/rally-openstack>`__
-openstack/sahara                         `https://opendev.org/openstack/sahara <https://opendev.org/openstack/sahara>`__
-openstack/sahara-dashboard               `https://opendev.org/openstack/sahara-dashboard <https://opendev.org/openstack/sahara-dashboard>`__
-openstack/senlin                         `https://opendev.org/openstack/senlin <https://opendev.org/openstack/senlin>`__
 openstack/shade                          `https://opendev.org/openstack/shade <https://opendev.org/openstack/shade>`__
 openstack/skyline-apiserver              `https://opendev.org/openstack/skyline-apiserver <https://opendev.org/openstack/skyline-apiserver>`__
-openstack/solum                          `https://opendev.org/openstack/solum <https://opendev.org/openstack/solum>`__
 openstack/storlets                       `https://opendev.org/openstack/storlets <https://opendev.org/openstack/storlets>`__
 openstack/tacker                         `https://opendev.org/openstack/tacker <https://opendev.org/openstack/tacker>`__
 openstack/tap-as-a-service               `https://opendev.org/openstack/tap-as-a-service <https://opendev.org/openstack/tap-as-a-service>`__
@@ -188,6 +174,7 @@ x/trio2o                                 `https://opendev.org/x/trio2o <https://
 x/valet                                  `https://opendev.org/x/valet <https://opendev.org/x/valet>`__
 x/vmware-nsx                             `https://opendev.org/x/vmware-nsx <https://opendev.org/x/vmware-nsx>`__
 x/vmware-vspc                            `https://opendev.org/x/vmware-vspc <https://opendev.org/x/vmware-vspc>`__
+x/whitebox-neutron-tempest-plugin        `https://opendev.org/x/whitebox-neutron-tempest-plugin <https://opendev.org/x/whitebox-neutron-tempest-plugin>`__
 ======================================== ===


@@ -238,14 +238,11 @@ package dependencies, packages may be listed at the following
 locations in the top-level of the plugin repository:

 - ``./devstack/files/debs/$plugin_name`` - Packages to install when running
-  on Ubuntu, Debian or Linux Mint.
+  on Ubuntu or Debian.

 - ``./devstack/files/rpms/$plugin_name`` - Packages to install when running
  on Red Hat, Fedora, or CentOS.

- ``./devstack/files/rpms-suse/$plugin_name`` - Packages to install when
-  running on SUSE Linux or openSUSE.
-
 Although there a no plans to remove this method of installing
 packages, plugins should consider it deprecated for ``bindep`` support
 described below.
@@ -0,0 +1,25 @@
+=======
+Tempest
+=======
+
+`Tempest`_ is the OpenStack Integration test suite. It is installed by default
+and is used to provide integration testing for many of the OpenStack services.
+Just like DevStack itself, it is possible to extend Tempest with plugins. In
+fact, many Tempest plugin packages also include DevStack plugin to do things
+like pre-create required static resources.
+
+The `Tempest documentation <Tempest>`_ provides a thorough guide to using
+Tempest. However, if you simply wish to run the standard set of Tempest tests
+against an existing deployment, you can do the following:
+
+.. code-block:: shell
+
+    cd /opt/stack/tempest
+    /opt/stack/data/venv/bin/tempest run ...
+
+The above assumes you have installed DevStack in the default location
+(configured via the ``DEST`` configuration variable) and have enabled
+virtualenv-based installation in the standard location (configured via the
+``USE_VENV`` and ``VENV_DEST`` configuration variables, respectively).
+
+.. _Tempest: https://docs.openstack.org/tempest/latest/
@@ -39,4 +39,5 @@
    CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined
 </VirtualHost>

+%WSGIPYTHONHOME%
 WSGISocketPrefix /var/run/%APACHE_NAME%
@@ -23,6 +23,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
 %SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%    SSLProtocol -all +TLSv1.3 +TLSv1.2
 %SSLLISTEN%</VirtualHost>

 Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
@@ -24,6 +24,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
 %SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%    SSLProtocol -all +TLSv1.3 +TLSv1.2
 %SSLLISTEN%</VirtualHost>

 Alias /networking %NEUTRON_BIN%/neutron-api
@@ -1,7 +1,5 @@
 conntrack
 curl
-dnsmasq-base
-dnsmasq-utils # for dhcp_release
 ebtables
 genisoimage # required for config_drive
 iptables
@@ -1,3 +0,0 @@
-enable-tftp
-tftp-root=/tftpboot
-dhcp-boot=pxelinux.0
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import os.path
+import json
+
+server_address = "/tmp/openstack.sock"
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+try:
+    sock.connect(server_address)
+except socket.error as msg:
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+
+    char = sock.recv(1)
+    if len(char) == 0:
+        print("Unexpected end of file", file=sys.stderr)
+        sys.exit(1)
+
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+        if len(char) == 0:
+            print("Unexpected end of file", file=sys.stderr)
+            sys.exit(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+try:
+    env = {}
+    passenv = ["CINDER_VERSION",
+               "OS_AUTH_URL",
+               "OS_NO_CACHE",
+               "OS_PASSWORD",
+               "OS_PROJECT_NAME",
+               "OS_REGION_NAME",
+               "OS_TENANT_NAME",
+               "OS_USERNAME",
+               "OS_VOLUME_API_VERSION",
+               "OS_CLOUD"]
+    for name in passenv:
+        if name in os.environ:
+            env[name] = os.environ[name]
+
+    cmd = {
+        "app": os.path.basename(sys.argv[0]),
+        "env": env,
+        "argv": sys.argv[1:]
+    }
+    try:
+        image_idx = sys.argv.index('image')
+        create_idx = sys.argv.index('create')
+        missing_file = image_idx < create_idx and \
+                not any(x.startswith('--file') for x in sys.argv)
+    except ValueError:
+        missing_file = False
+
+    if missing_file:
+        # This means we were called with an image create command, but were
+        # not provided a --file option. That likely means we're being passed
+        # the image data to stdin, which won't work because we do not proxy
+        # stdin to the server. So, we just reject the operation and ask the
+        # caller to provide the file with --file instead.
+        # We've already connected to the server, we need to send it some dummy
+        # data so it doesn't wait forever.
+        send(sock, {})
+        print('Image create without --file is not allowed in server mode',
+              file=sys.stderr)
+        sys.exit(1)
+    else:
+        send(sock, cmd)
+
+    doc = recv(sock)
+    if doc["stdout"] != b'':
+        print(doc["stdout"], end='')
+    if doc["stderr"] != b'':
+        print(doc["stderr"], file=sys.stderr)
+    sys.exit(doc["status"])
+finally:
+    sock.close()
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import json
+
+from openstackclient import shell as osc_shell
+from io import StringIO
+
+server_address = "/tmp/openstack.sock"
+
+try:
+    os.unlink(server_address)
+except OSError:
+    if os.path.exists(server_address):
+        raise
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+print('starting up on %s' % server_address, file=sys.stderr)
+sock.bind(server_address)
+
+# Listen for incoming connections
+sock.listen(1)
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+    char = sock.recv(1)
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+while True:
+    csock, client_address = sock.accept()
+    try:
+        doc = recv(csock)
+
+        print("%s %s" % (doc["app"], doc["argv"]), file=sys.stderr)
+        oldenv = {}
+        for name in doc["env"].keys():
+            oldenv[name] = os.environ.get(name, None)
+            os.environ[name] = doc["env"][name]
+
+        try:
+            old_stdout = sys.stdout
+            old_stderr = sys.stderr
+            my_stdout = sys.stdout = StringIO()
+            my_stderr = sys.stderr = StringIO()
+
+            class Exit(BaseException):
+                def __init__(self, status):
+                    self.status = status
+
+            def noexit(stat):
+                raise Exit(stat)
+
+            sys.exit = noexit
+
+            if doc["app"] == "openstack":
+                sh = osc_shell.OpenStackShell()
+                ret = sh.run(doc["argv"])
+            else:
+                print("Unknown application %s" % doc["app"], file=sys.stderr)
+                ret = 1
+        except Exit as e:
+            ret = e.status
+        finally:
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+
+            for name in oldenv.keys():
+                if oldenv[name] is None:
+                    del os.environ[name]
+                else:
+                    os.environ[name] = oldenv[name]
+
+        send(csock, {
+            "stdout": my_stdout.getvalue(),
+            "stderr": my_stderr.getvalue(),
+            "status": ret,
+        })
+
+    except BaseException as e:
+        print(e, file=sys.stderr)
+    finally:
+        csock.close()
@@ -1 +0,0 @@
-dnsmasq
@@ -1,3 +0,0 @@
-ceph    # NOPRIME
-lsb
-xfsprogs
@@ -1,3 +0,0 @@
-lvm2
-qemu-tools
-tgt # NOPRIME
@@ -1 +0,0 @@
-dstat
@@ -1,34 +0,0 @@
-apache2
-apache2-devel
-bc
-ca-certificates-mozilla
-curl
-gawk
-gcc
-gcc-c++
-git-core
-graphviz # docs
-iputils
-libffi-devel  # pyOpenSSL
-libjpeg8-devel # Pillow 3.0.0
-libopenssl-devel # to rebuild pyOpenSSL if needed
-libxslt-devel  # lxml
-lsof # useful when debugging
-make
-net-tools
-openssh
-openssl
-pcre-devel # python-pcre
-postgresql-devel  # psycopg2
-psmisc
-python3-systemd
-python-cmd2 # dist:opensuse-12.3
-python-devel  # pyOpenSSL
-python-xml
-tar
-tcpdump
-unzip
-util-linux
-wget
-which
-zlib-devel
@@ -1,2 +0,0 @@
-apache2-mod_wsgi  # NOPRIME
-apache2  # NOPRIME
@@ -1,4 +0,0 @@
-cyrus-sasl-devel
-memcached
-openldap2-devel
-sqlite3
@@ -1,3 +0,0 @@
-openldap2
-openldap2-client
-python-ldap
@@ -1 +0,0 @@
-python-dateutil
@@ -1,10 +0,0 @@
-cdrkit-cdrtools-compat # dist:sle12
-cryptsetup
-dosfstools
-libosinfo
-lvm2
-mkisofs # not:sle12
-open-iscsi
-sg3_utils
-# Stuff for diablo volumes
-sysfsutils
@@ -1 +0,0 @@
-ipset
@@ -1,12 +0,0 @@
-acl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
-ebtables
-haproxy # to serve as metadata proxy inside router/dhcp namespaces
-iptables
-iputils
-rabbitmq-server # NOPRIME
-radvd # NOPRIME
-sqlite3
-sudo
-vlan
@@ -1,2 +0,0 @@
-conntrack-tools
-keepalived
@@ -1,23 +0,0 @@
-cdrkit-cdrtools-compat # dist:sle12
-conntrack-tools
-curl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
-ebtables
-iptables
-iputils
-kpartx
-kvm # NOPRIME
-libvirt # NOPRIME
-libvirt-python # NOPRIME
-# mkisofs is required for config_drive
-mkisofs # not:sle12
-parted
-polkit
-# qemu as fallback if kvm cannot be used
-qemu # NOPRIME
-rabbitmq-server # NOPRIME
-socat
-sqlite3
-sudo
-vlan
@@ -1,3 +0,0 @@
-
-openvswitch
-openvswitch-switch
@@ -1,2 +0,0 @@
-lsscsi
-open-iscsi
@@ -1 +0,0 @@
-neutron-agent
@@ -1 +0,0 @@
-neutron-l3
@@ -1,6 +0,0 @@
-curl
-liberasurecode-devel
-memcached
-sqlite3
-xfsprogs
-xinetd
@@ -1,3 +1,3 @@
 ceph    # NOPRIME
-redhat-lsb-core # not:rhel9,openEuler-20.03
+redhat-lsb-core # not:rhel9,openEuler-22.03
 xfsprogs
@@ -6,9 +6,11 @@ gcc
 gcc-c++
 gettext  # used for compiling message catalogs
 git-core
+glibc-langpack-en # dist:rhel9
 graphviz # needed only for docs
 httpd
 httpd-devel
+iptables-nft # dist:rhel9
 iptables-services
 java-1.8.0-openjdk-headless
 libffi-devel
@@ -16,7 +18,7 @@ libjpeg-turbo-devel # Pillow 3.0.0
 libxml2-devel # lxml
 libxslt-devel # lxml
 libyaml-devel
-make # dist:openEuler-20.03
+mod_ssl # required for tls-proxy on centos 9 stream computes
 net-tools
 openssh-server
 openssl
@@ -26,10 +28,9 @@ pkgconfig
 postgresql-devel  # psycopg2
 psmisc
 python3-devel
-python3-pip
+python3-pip # not:openEuler-22.03
 python3-systemd
-redhat-rpm-config # not:openEuler-20.03    missing dep for gcc hardening flags, see rhbz#1217376
-systemd-devel # dist:openEuler-20.03
+redhat-rpm-config # not:openEuler-22.03  missing dep for gcc hardening flags, see rhbz#1217376
 tar
 tcpdump
 unzip
@@ -1,12 +1,10 @@
 conntrack-tools
 curl
-dnsmasq # for q-dhcp
-dnsmasq-utils # for dhcp_release
 ebtables
 genisoimage # not:rhel9 required for config_drive
 iptables
 iputils
-kernel-modules # not:openEuler-20.03
+kernel-modules # not:openEuler-22.03
 kpartx
 parted
 polkit
@@ -1,7 +1,7 @@
 curl
-liberasurecode-devel # not:openEuler-20.03
+liberasurecode-devel
 memcached
 rsync-daemon
 sqlite
 xfsprogs
-xinetd # not:f35,rhel9
+xinetd # not:f36,rhel9
@@ -118,7 +118,7 @@ function _upload_image {
        useimport="--import"
    fi

-    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties < "${image}"
+    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties --file $(readlink -f "${image}")
 }

 # Retrieve an image from a URL and upload into Glance.
@@ -133,17 +133,28 @@ function upload_image {

    local image image_fname image_name

+    local max_attempts=5
+
    # Create a directory for the downloaded image tarballs.
    mkdir -p $FILES/images
    image_fname=`basename "$image_url"`
    if [[ $image_url != file* ]]; then
        # Downloads the image (uec ami+akistyle), then extracts it.
        if [[ ! -f $FILES/$image_fname || "$(stat -c "%s" $FILES/$image_fname)" = "0" ]]; then
-            wget --progress=dot:giga -c $image_url -O $FILES/$image_fname
-            if [[ $? -ne 0 ]]; then
-                echo "Not found: $image_url"
-                return
-            fi
+            for attempt in `seq $max_attempts`; do
+                local rc=0
+                wget --progress=dot:giga -c $image_url -O $FILES/$image_fname || rc=$?
+                if [[ $rc -ne 0 ]]; then
+                    if [[ "$attempt" -eq "$max_attempts" ]]; then
+                        echo "Not found: $image_url"
+                        return
+                    fi
+                    echo "Download failed, retrying in $attempt second, attempt: $attempt"
+                    sleep $attempt
+                else
+                    break
+                fi
+            done
        fi
        image="$FILES/${image_fname}"
    else
@@ -414,10 +425,10 @@ function upload_image {
        # kernel for use when uploading the root filesystem.
        local kernel_id="" ramdisk_id="";
        if [ -n "$kernel" ]; then
-            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki < "$kernel" | grep ' id ' | get_field 2)
+            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki --file $(readlink -f "$kernel") -f value -c id)
        fi
        if [ -n "$ramdisk" ]; then
-            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari < "$ramdisk" | grep ' id ' | get_field 2)
+            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari --file $(readlink -f "$ramdisk") -f value -c id)
        fi
        _upload_image "${image_name%.img}" ami ami "$image" ${kernel_id:+ kernel_id=$kernel_id} ${ramdisk_id:+ ramdisk_id=$ramdisk_id} $img_property
    fi
@@ -683,6 +694,8 @@ function setup_colorized_logging {
    iniset $conf_file DEFAULT logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_debug_format_suffix "[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s [01;35m%(instance)s[00m"
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_systemd_logging {
@@ -704,6 +717,9 @@ function setup_systemd_logging {
    iniset $conf_file DEFAULT logging_context_format_string "%(color)s%(levelname)s %(name)s [[01;36m%(global_request_id)s %(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_default_format_string "%(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "ERROR %(name)s [01;35m%(instance)s[00m"
+
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_standard_logging_identity {
@@ -49,7 +49,7 @@ KILL_PATH="$(which kill)"
 STACK_ENV_VARS="BASE_SQL_CONN DATA_DIR DEST ENABLED_SERVICES HOST_IP \
    KEYSTONE_SERVICE_URI \
    LOGFILE OS_CACERT SERVICE_HOST STACK_USER TLS_IP \
-    HOST_IPV6 SERVICE_IP_VERSION"
+    HOST_IPV6 SERVICE_IP_VERSION TUNNEL_ENDPOINT_IP TUNNEL_IP_VERSION"


 # Saves significant environment variables to .stackenv for later use
@@ -236,6 +236,27 @@ function trueorfalse {
    $xtrace
 }

+# bool_to_int <True|False>
+#
+# Convert True|False to int 1 or 0
+# This function can be used to convert the output of trueorfalse
+# to an int follow c conventions where false is 0 and 1 it true.
+function bool_to_int {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    if [ -z $1 ]; then
+        die $LINENO "Bool value required"
+    fi
+    if [[ $1 == "True" ]] ; then
+        echo '1'
+    else
+        echo '0'
+    fi
+    $xtrace
+}
+
+
 function isset {
    [[ -v "$1" ]]
 }
@@ -380,9 +401,9 @@ function warn {
 # such as "install_package" further abstract things in better ways.
 #
 # ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
-# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
+# ``os_RELEASE`` - major release: ``22.04`` (Ubuntu), ``23`` (Fedora)
 # ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
-# ``os_CODENAME`` - vendor's codename for release: ``xenial``
+# ``os_CODENAME`` - vendor's codename for release: ``jammy``

 declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME

@@ -412,12 +433,15 @@ function _ensure_lsb_release {
 #  - os_VENDOR
 #  - os_PACKAGE
 function GetOSVersion {
-    # CentOS Stream 9 does not provide lsb_release
+    # CentOS Stream 9 and RHEL 9 do not provide lsb_release
    source /etc/os-release
-    if [[ "${ID}${VERSION}" == "centos9" ]]; then
+    if [[ "${ID}${VERSION}" == "centos9" ]] || [[ "${ID}${VERSION}" =~ "rhel9" ]]; then
        os_RELEASE=${VERSION_ID}
        os_CODENAME="n/a"
        os_VENDOR=$(echo $NAME | tr -d '[:space:]')
+    elif [[ "${ID}${VERSION}" =~ "rocky9" ]]; then
+        os_VENDOR="Rocky"
+        os_RELEASE=${VERSION_ID}
    else
        _ensure_lsb_release

@@ -426,7 +450,7 @@ function GetOSVersion {
        os_VENDOR=$(lsb_release -i -s)
    fi

-    if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
+    if [[ $os_VENDOR =~ (Debian|Ubuntu) ]]; then
        os_PACKAGE="deb"
    else
        os_PACKAGE="rpm"
@@ -444,36 +468,24 @@ declare -g DISTRO

 function GetDistro {
    GetOSVersion
-    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
-            "$os_VENDOR" =~ (LinuxMint) ]]; then
-        # 'Everyone' refers to Ubuntu / Debian / Mint releases by
+    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
+        # 'Everyone' refers to Ubuntu / Debian releases by
        # the code name adjective
        DISTRO=$os_CODENAME
    elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
        # For Fedora, just use 'f' and the release
        DISTRO="f$os_RELEASE"
-    elif is_opensuse; then
-        DISTRO="opensuse-$os_RELEASE"
-        # Tumbleweed uses "n/a" as a codename, and the release is a datestring
-        # like 20180218, so not very useful. Leap however uses a release
-        # with a "dot", so for example 15.0
-        [ "$os_CODENAME" = "n/a" -a "$os_RELEASE" = "${os_RELEASE/\./}" ] && \
-            DISTRO="opensuse-tumbleweed"
-    elif is_suse_linux_enterprise; then
-        # just use major release
-        DISTRO="sle${os_RELEASE%.*}"
    elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
        "$os_VENDOR" =~ (CentOS) || \
        "$os_VENDOR" =~ (AlmaLinux) || \
        "$os_VENDOR" =~ (Scientific) || \
        "$os_VENDOR" =~ (OracleServer) || \
+        "$os_VENDOR" =~ (Rocky) || \
        "$os_VENDOR" =~ (Virtuozzo) ]]; then
        # Drop the . release as we assume it's compatible
        # XXX re-evaluate when we get RHEL10
        DISTRO="rhel${os_RELEASE::1}"
    elif [[ "$os_VENDOR" =~ (openEuler) ]]; then
-        # The DISTRO here is `openEuler-20.03`. While, actually only openEuler
-        # 20.03 LTS SP2 is fully tested. Other SP version maybe have bugs.
        DISTRO="openEuler-$os_RELEASE"
    else
        # We can't make a good choice here.  Setting a sensible DISTRO
@@ -518,7 +530,7 @@ function is_oraclelinux {


 # Determine if current distribution is a Fedora-based distribution
-# (Fedora, RHEL, CentOS, etc).
+# (Fedora, RHEL, CentOS, Rocky, etc).
 # is_fedora
 function is_fedora {
    if [[ -z "$os_VENDOR" ]]; then
@@ -529,43 +541,14 @@ function is_fedora {
        [ "$os_VENDOR" = "openEuler" ] || \
        [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
        [ "$os_VENDOR" = "RedHatEnterprise" ] || \
+        [ "$os_VENDOR" = "RedHatEnterpriseLinux" ] || \
+        [ "$os_VENDOR" = "Rocky" ] || \
        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \
        [ "$os_VENDOR" = "AlmaLinux" ] || \
        [ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ]
 }


-# Determine if current distribution is a SUSE-based distribution
-# (openSUSE, SLE).
-# is_suse
-function is_suse {
-    is_opensuse || is_suse_linux_enterprise
-}
-
-
-# Determine if current distribution is an openSUSE distribution
-# is_opensuse
-function is_opensuse {
-    if [[ -z "$os_VENDOR" ]]; then
-        GetOSVersion
-    fi
-
-    [[ "$os_VENDOR" =~ (openSUSE) ]]
-}
-
-
-# Determine if current distribution is a SUSE Linux Enterprise (SLE)
-# distribution
-# is_suse_linux_enterprise
-function is_suse_linux_enterprise {
-    if [[ -z "$os_VENDOR" ]]; then
-        GetOSVersion
-    fi
-
-    [[ "$os_VENDOR" =~ (^SUSE) ]]
-}
-
-
 # Determine if current distribution is an Ubuntu-based distribution
 # It will also detect non-Ubuntu but Debian-based distros
 # is_ubuntu
@@ -576,6 +559,8 @@ function is_ubuntu {
    [ "$os_PACKAGE" = "deb" ]
 }

+# Determine if current distribution is an openEuler distribution
+# is_openeuler
 function is_openeuler {
    if [[ -z "$os_PACKAGE" ]]; then
        GetOSVersion
@@ -646,8 +631,10 @@ function git_clone {
                echo "the project to the \$PROJECTS variable in the job definition."
                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
-            # '--branch' can also take tags
-            git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref
+            git_timed clone --no-checkout $git_clone_flags $git_remote $git_dest
+            cd $git_dest
+            git_timed fetch $git_clone_flags origin $git_ref
+            git_timed checkout FETCH_HEAD
        elif [[ "$RECLONE" = "True" ]]; then
            # if it does exist then simulate what clone does if asked to RECLONE
            cd $git_dest
@@ -657,7 +644,7 @@ function git_clone {
            # remove the existing ignored files (like pyc) as they cause breakage
            # (due to the py files having older timestamps than our pyc, so python
            # thinks the pyc files are correct using them)
-            find $git_dest -name '*.pyc' -delete
+            sudo find $git_dest -name '*.pyc' -delete

            # handle git_ref accordingly to type (tag, branch)
            if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
@@ -673,6 +660,18 @@ function git_clone {
        fi
    fi

+    # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
+    # about how we clone and work with repos.  Mark them safe globally
+    # as a work-around.
+    #
+    # NOTE(danms): On bionic (and likely others) git-config may write
+    # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+    # writes these changes to /etc/gitconfig which is more
+    # discoverable anyway.
+    #
+    # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
+    sudo git config --system --add safe.directory ${git_dest}
+
    # print out the results so we know what change was used in the logs
    cd $git_dest
    git show --oneline | head -1
@@ -772,7 +771,7 @@ function get_default_host_ip {
    if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
        host_ip=""
        # Find the interface used for the default route
-        host_ip_iface=${host_ip_iface:-$(ip -f $af route | awk '/default/ {print $5}' | head -1)}
+        host_ip_iface=${host_ip_iface:-$(ip -f $af route list match default table all | grep via | awk '/default/ {print $5}' | head -1)}
        local host_ips
        host_ips=$(LC_ALL=C ip -f $af addr show ${host_ip_iface} | sed /temporary/d |awk /$af'/ {split($2,parts,"/");  print parts[1]}')
        local ip
@@ -875,14 +874,9 @@ function policy_add {
 # Usage: get_or_create_domain <name> <description>
 function get_or_create_domain {
    local domain_id
-    # Gets domain id
    domain_id=$(
-        # Gets domain id
-        openstack --os-cloud devstack-system-admin domain show $1 \
-            -f value -c id 2>/dev/null ||
-        # Creates new domain
        openstack --os-cloud devstack-system-admin domain create $1 \
-            --description "$2" \
+            --description "$2" --or-show \
            -f value -c id
    )
    echo $domain_id
@@ -971,29 +965,22 @@ function _get_domain_args {
 # Usage: get_or_add_user_project_role <role> <user> <project> [<user_domain> <project_domain>]
 function get_or_add_user_project_role {
    local user_role_id
+    local domain_args

    domain_args=$(_get_domain_args $4 $5)

-    # Gets user role id
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack --os-cloud devstack-system-admin role add $1 \
+        --user $2 \
+        --project $3 \
+        $domain_args
    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --project $3 \
        $domain_args \
-        | grep '^|\s[a-f0-9]\+' | get_field 1)
-    if [[ -z "$user_role_id" ]]; then
-        # Adds role to user and get it
-        openstack --os-cloud devstack-system-admin role add $1 \
-            --user $2 \
-            --project $3 \
-            $domain_args
-        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
-            --role $1 \
-            --user $2 \
-            --project $3 \
-            $domain_args \
-            | grep '^|\s[a-f0-9]\+' | get_field 1)
-    fi
+        -c Role -f value)
    echo $user_role_id
 }

@@ -1001,23 +988,18 @@ function get_or_add_user_project_role {
 # Usage: get_or_add_user_domain_role <role> <user> <domain>
 function get_or_add_user_domain_role {
    local user_role_id
-    # Gets user role id
+
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack --os-cloud devstack-system-admin role add $1 \
+        --user $2 \
+        --domain $3
    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --domain $3 \
-        | grep '^|\s[a-f0-9]\+' | get_field 1)
-    if [[ -z "$user_role_id" ]]; then
-        # Adds role to user and get it
-        openstack --os-cloud devstack-system-admin role add $1 \
-            --user $2 \
-            --domain $3
-        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
-            --role $1 \
-            --user $2 \
-            --domain $3 \
-            | grep '^|\s[a-f0-9]\+' | get_field 1)
-    fi
+        -c Role -f value)
+
    echo $user_role_id
 }

@@ -1056,23 +1038,18 @@ function get_or_add_user_system_role {
 # Usage: get_or_add_group_project_role <role> <group> <project>
 function get_or_add_group_project_role {
    local group_role_id
-    # Gets group role id
+
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack role add $1 \
+        --group $2 \
+        --project $3
    group_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --group $2 \
        --project $3 \
-        -f value)
-    if [[ -z "$group_role_id" ]]; then
-        # Adds role to group and get it
-        openstack --os-cloud devstack-system-admin role add $1 \
-            --group $2 \
-            --project $3
-        group_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
-            --role $1 \
-            --group $2 \
-            --project $3 \
-            -f value)
-    fi
+        -f value -c Role)
+
    echo $group_role_id
 }

@@ -1154,7 +1131,13 @@ function is_ironic_hardware {
 }

 function is_ironic_enforce_scope {
-    is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" ]] && return 0
+    is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]] && return 0
+    return 1
+}
+
+function is_ironic_sharded {
+    # todo(JayF): Support >1 shard with multiple n-cpu instances for each
+    is_service_enabled ironic && [[ "$IRONIC_SHARDS" == "1" ]] && return 0
    return 1
 }

@@ -1174,8 +1157,6 @@ function _get_package_dir {
        pkg_dir=$base_dir/debs
    elif is_fedora; then
        pkg_dir=$base_dir/rpms
-    elif is_suse; then
-        pkg_dir=$base_dir/rpms-suse
    else
        exit_distro_not_supported "list of packages"
    fi
@@ -1450,8 +1431,6 @@ function real_install_package {
        apt_get install "$@"
    elif is_fedora; then
        yum_install "$@"
-    elif is_suse; then
-        zypper_install "$@"
    else
        exit_distro_not_supported "installing packages"
    fi
@@ -1493,8 +1472,6 @@ function uninstall_package {
        apt_get purge "$@"
    elif is_fedora; then
        sudo dnf remove -y "$@" ||:
-    elif is_suse; then
-        sudo zypper remove -y "$@" ||:
    else
        exit_distro_not_supported "uninstalling packages"
    fi
@@ -1563,6 +1540,7 @@ function write_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local extra=""
    if [[ -n "$group" ]]; then
        extra="Group=$group"
@@ -1571,11 +1549,15 @@ function write_user_unit_file {
    mkdir -p $SYSTEMD_DIR

    iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
    iniset -sudo $unitfile "Service" "User" "$user"
    iniset -sudo $unitfile "Service" "ExecStart" "$command"
    iniset -sudo $unitfile "Service" "KillMode" "process"
    iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
    iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1590,10 +1572,12 @@ function write_uwsgi_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local unitfile="$SYSTEMD_DIR/$service"
    mkdir -p $SYSTEMD_DIR

    iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
    iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
    iniset -sudo $unitfile "Service" "User" "$user"
    iniset -sudo $unitfile "Service" "ExecStart" "$command"
@@ -1604,6 +1588,9 @@ function write_uwsgi_user_unit_file {
    iniset -sudo $unitfile "Service" "NotifyAccess" "all"
    iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"

+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1651,10 +1638,17 @@ function _run_under_systemd {
    local systemd_service="devstack@$service.service"
    local group=$3
    local user=${4:-$STACK_USER}
+    if [[ -z "$user" ]]; then
+        user=$STACK_USER
+    fi
+    local env_vars="$5"
    if [[ "$command" =~ "uwsgi" ]] ; then
-        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        if [[ "$GLOBAL_VENV" == "True" ]] ; then
+            cmd="$cmd --venv $DEVSTACK_VENV"
+        fi
+        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    else
-        write_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    fi

    $SYSTEMCTL enable $systemd_service
@@ -1675,18 +1669,20 @@ function is_running {
 # If the command includes shell metachatacters (;<>*) it must be run using a shell
 # If an optional group is provided sg will be used to run the
 # command as that group.
-# run_process service "command-line" [group] [user]
+# run_process service "command-line" [group] [user] [env_vars]
+# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2"
 function run_process {
    local service=$1
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"

    local name=$service

    time_start "run_process"
    if is_service_enabled $service; then
-        _run_under_systemd "$name" "$command" "$group" "$user"
+        _run_under_systemd "$name" "$command" "$group" "$user" "$env_vars"
    fi
    time_stop "run_process"
 }
@@ -2442,6 +2438,11 @@ function time_stop {
    _TIME_TOTAL[$name]=$(($total + $elapsed_time))
 }

+function install_openstack_cli_server {
+    export PATH=$TOP_DIR/files/openstack-cli-server:$PATH
+    run_process openstack-cli-server "$PYTHON $TOP_DIR/files/openstack-cli-server/openstack-cli-server"
+}
+
 function oscwrap {
    local xtrace
    xtrace=$(set +o | grep xtrace)
@@ -2537,6 +2538,11 @@ function clean_pyc_files {
    fi
 }

+function is_fips_enabled {
+    fips=`cat /proc/sys/crypto/fips_enabled`
+    [ "$fips" == "1" ]
+}
+
 # Restore xtrace
 $_XTRACE_FUNCTIONS_COMMON

@@ -189,6 +189,9 @@ function iniset {
    local option=$3
    local value=$4

+    # Escape the ampersand character (&)
+    value=$(echo $value | sed -e 's/&/\\&/g')
+
    if [[ -z $section || -z $option ]]; then
        $xtrace
        return
@@ -7,7 +7,6 @@
 # External functions used:
 # - GetOSVersion
 # - is_fedora
-# - is_suse
 # - safe_chown

 # Save trace setting
@@ -33,6 +32,26 @@ function join_extras {
 # Python Functions
 # ================

+# Setup the global devstack virtualenvs and the associated environment
+# updates.
+function setup_devstack_virtualenv {
+    # We run devstack out of a global virtualenv.
+    if [[ ! -d $DEVSTACK_VENV ]] ; then
+        # Using system site packages to enable nova to use libguestfs.
+        # This package is currently installed via the distro and not
+        # available on pypi.
+        python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
+        pip_install -U pip setuptools
+        #NOTE(rpittau): workaround for simplejson removal in osc
+        #  https://review.opendev.org/c/openstack/python-openstackclient/+/920001
+        pip_install -U simplejson
+    fi
+    if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
+        export PATH="$DEVSTACK_VENV/bin:$PATH"
+        export PYTHON="$DEVSTACK_VENV/bin/python3"
+    fi
+}
+
 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
@@ -61,9 +80,11 @@ function get_python_exec_prefix {
    fi
    $xtrace

-    local PYTHON_PATH=/usr/local/bin
-    is_suse && PYTHON_PATH=/usr/bin
-    echo $PYTHON_PATH
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        echo "$DEVSTACK_VENV/bin"
+    else
+        echo "/usr/local/bin"
+    fi
 }

 # Wrapper for ``pip install`` that only installs versions of libraries
@@ -168,6 +189,14 @@ function pip_install {
    if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
        local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
        local sudo_pip="env"
+    elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
+        # We have to check that the DEVSTACK_VENV exists because early
+        # devstack boostrapping needs to operate in a system context
+        # too bootstrap pip. Once pip is bootstrapped we create the
+        # global venv and can start to use it.
+        local cmd_pip=$DEVSTACK_VENV/bin/pip
+        local sudo_pip="env"
+        echo "Using python $PYTHON3_VERSION to install $package_dir"
    else
        local cmd_pip="python$PYTHON3_VERSION -m pip"
        # See
@@ -186,15 +215,11 @@ function pip_install {

    $xtrace

-    # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
-    # the same behaviour of setuptools before version 25.0.0.
-    # related issue: https://github.com/pypa/pip/issues/3874
    $sudo_pip \
        http_proxy="${http_proxy:-}" \
        https_proxy="${https_proxy:-}" \
        no_proxy="${no_proxy:-}" \
        PIP_FIND_LINKS=$PIP_FIND_LINKS \
-        SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
        $cmd_pip $upgrade \
        $@
    result=$?
@@ -383,6 +408,9 @@ function _setup_package_with_constraints_edit {
        # source we are about to do.
        local name
        name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
+        if [ -z $name ]; then
+            name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
+        fi
        $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
            $REQUIREMENTS_DIR/upper-constraints.txt -- $name
    fi
@@ -445,8 +473,11 @@ function setup_package {

    pip_install $flags "$project_dir$extras"
    # ensure that further actions can do things like setup.py sdist
-    if [[ "$flags" == "-e" ]]; then
-        safe_chown -R $STACK_USER $1/*.egg-info
+    if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
+        # egg-info is not created when project have pyproject.toml
+        if [ -d $1/*.egg-info ]; then
+            safe_chown -R $STACK_USER $1/*.egg-info
+        fi
    fi
 }

@@ -466,8 +497,6 @@ function install_python {
 function install_python3 {
    if is_ubuntu; then
        apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
-    elif is_suse; then
-        install_package python3-devel python3-dbm
    elif is_fedora; then
        if [ "$os_VENDOR" = "Fedora" ]; then
            install_package python${PYTHON3_VERSION//.}
@@ -60,6 +60,11 @@ function configure_rootwrap {
    sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf
    sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf

+    # Rely on $PATH set by devstack to determine what is safe to execute
+    # by rootwrap rather than use explicit whitelist of paths in
+    # rootwrap.conf
+    sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf
+
    # Set up the rootwrap sudoers
    local tempfile
    tempfile=$(mktemp)
@@ -44,10 +44,6 @@ elif is_fedora; then
    APACHE_NAME=httpd
    APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/conf.d}
    APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
-elif is_suse; then
-    APACHE_NAME=apache2
-    APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/vhosts.d}
-    APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
 fi
 APACHE_LOG_DIR="/var/log/${APACHE_NAME}"

@@ -65,11 +61,6 @@ function enable_apache_mod {
            sudo a2enmod $mod
            restart_apache_server
        fi
-    elif is_suse; then
-        if ! a2enmod -q $mod ; then
-            sudo a2enmod $mod
-            restart_apache_server
-        fi
    elif is_fedora; then
        # pass
        true
@@ -95,7 +86,7 @@ function install_apache_uwsgi {
    # didn't fix Python 3.10 compatibility before release.  Should be
    # fixed in uwsgi 4.9.0; can remove this when packages available
    # or we drop this release
-    elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ f35 ]]; then
+    elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ f36 ]]; then
        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
        # default; the mod_proxy_uwsgi package actually conflicts now.
        # See:
@@ -104,10 +95,6 @@ function install_apache_uwsgi {
        # Thus there is nothing else to do after this install
        install_package uwsgi \
                        uwsgi-plugin-python3
-    elif [[ $os_VENDOR =~ openSUSE ]]; then
-        install_package uwsgi \
-                        uwsgi-python3 \
-                        apache2-mod_uwsgi
    else
        # Compile uwsgi from source.
        local dir
@@ -125,7 +112,7 @@ function install_apache_uwsgi {
        sudo rm -rf $dir
    fi

-    if is_ubuntu || is_suse ; then
+    if is_ubuntu; then
        # we've got to enable proxy and proxy_uwsgi for this to work
        sudo a2enmod proxy
        sudo a2enmod proxy_uwsgi
@@ -150,13 +137,13 @@ function install_apache_wsgi {
    elif is_fedora; then
        sudo rm -f /etc/httpd/conf.d/000-*
        install_package httpd python3-mod_wsgi
+        # rpm distros dont enable httpd by default so enable it to support reboots.
+        sudo systemctl enable httpd
        # For consistency with Ubuntu, switch to the worker mpm, as
        # the default is event
        sudo sed -i '/mod_mpm_prefork.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
        sudo sed -i '/mod_mpm_event.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
        sudo sed -i '/mod_mpm_worker.so/s/^#//g' /etc/httpd/conf.modules.d/00-mpm.conf
-    elif is_suse; then
-        install_package apache2 apache2-mod_wsgi
    else
        exit_distro_not_supported "apache wsgi installation"
    fi
@@ -171,7 +158,7 @@ function install_apache_wsgi {
 # recognise it. a2ensite and a2dissite ignore the .conf suffix used as parameter. The default sites'
 # files are 000-default.conf and default-ssl.conf.
 #
-# On Fedora and openSUSE, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
+# On Fedora, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
 #
 # On RHEL and CentOS, things should hopefully work as in Fedora.
 #
@@ -187,7 +174,7 @@ function apache_site_config_for {
    if is_ubuntu; then
        # Ubuntu 14.04 - Apache 2.4
        echo $APACHE_CONF_DIR/${site}.conf
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        # fedora conf.d is only imported if it ends with .conf so this is approx the same
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        if [ -f $enabled_site_file ]; then
@@ -205,7 +192,7 @@ function enable_apache_site {
    enable_apache_mod version
    if is_ubuntu; then
        sudo a2ensite ${site}
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        # Do nothing if site already enabled or no site config exists
        if [[ -f ${enabled_site_file}.disabled ]] && [[ ! -f ${enabled_site_file} ]]; then
@@ -219,7 +206,7 @@ function disable_apache_site {
    local site=$@
    if is_ubuntu; then
        sudo a2dissite ${site} || true
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        # Do nothing if no site config exists
        if [[ -f ${enabled_site_file} ]]; then
@@ -250,13 +237,17 @@ function restart_apache_server {
    restart_service $APACHE_NAME
 }

+# write_uwsgi_config() - Create a new uWSGI config file
 function write_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
    local http=$4
-    local name=""
-    name=$(basename $wsgi)
+    local name=$5
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.
@@ -271,39 +262,47 @@ function write_uwsgi_config {
    local socket="$socket_dir/${name}.socket"

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
-    iniset "$file" uwsgi processes $API_WORKERS
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given. Note that the regex isn't exhaustive - neither Python modules
+    # nor Python variables can start with a number - but it's "good enough"
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,python3
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
+    iniset "$conf" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
+    iniset "$conf" uwsgi lazy-apps true

    # If we said bind directly to http, then do that and don't start the apache proxy
    if [[ -n "$http" ]]; then
-        iniset "$file" uwsgi http $http
+        iniset "$conf" uwsgi http $http
    else
        local apache_conf=""
        apache_conf=$(apache_site_config_for $name)
-        iniset "$file" uwsgi socket "$socket"
-        iniset "$file" uwsgi chmod-socket 666
-        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 " | sudo tee -a $apache_conf
+        iniset "$conf" uwsgi socket "$socket"
+        iniset "$conf" uwsgi chmod-socket 666
+        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 acquire=1 " | sudo tee -a $apache_conf
        enable_apache_site $name
        restart_apache_server
    fi
@@ -316,47 +315,58 @@ function write_uwsgi_config {
 # but that involves having apache buffer the request before sending it to
 # uwsgi.
 function write_local_uwsgi_http_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
-    name=$(basename $wsgi)
+    local name=$4
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
    port=$(get_random_port)
-    iniset "$file" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
-    iniset "$file" uwsgi processes $API_WORKERS
+    iniset "$conf" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,python3
-    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
-    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
+    # uwsgi recommends this to prevent thundering herd on accept.
+    iniset "$conf" uwsgi thunder-lock true
+    # Set hook to trigger graceful shutdown on SIGTERM
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
-    iniset "$file" uwsgi chmod-socket 666
-    iniset "$file" uwsgi http-raw-body true
-    iniset "$file" uwsgi http-chunked-input true
-    iniset "$file" uwsgi http-auto-chunked true
-    iniset "$file" uwsgi http-keepalive false
+    iniset "$conf" uwsgi lazy-apps true
+    iniset "$conf" uwsgi chmod-socket 666
+    iniset "$conf" uwsgi http-raw-body true
+    iniset "$conf" uwsgi http-chunked-input true
+    iniset "$conf" uwsgi http-auto-chunked true
+    iniset "$conf" uwsgi http-keepalive false
    # Increase socket timeout for slow chunked uploads
-    iniset "$file" uwsgi socket-timeout 30
+    iniset "$conf" uwsgi socket-timeout 30

    enable_apache_mod proxy
    enable_apache_mod proxy_http
@@ -364,7 +374,7 @@ function write_local_uwsgi_http_config {
    apache_conf=$(apache_site_config_for $name)
    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${url}\" \"http://$APACHE_LOCAL_HOST:$port\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${url}\" \"http://$APACHE_LOCAL_HOST:$port\" retry=0 acquire=1 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }
@@ -383,18 +393,24 @@ function write_local_proxy_http_config {

    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${loc}\" \"$url\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${loc}\" \"$url\" retry=0 acquire=1 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }

 function remove_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local name=""
+    # TODO(stephenfin): Remove this call when everyone is using module path
+    # configuration instead of file path configuration
    name=$(basename $wsgi)

-    rm -rf $file
+    if [[ "$wsgi" = /* ]]; then
+        deprecated "Passing a wsgi script to remove_uwsgi_config is deprecated, pass an application name instead"
+    fi
+
+    rm -rf $conf
    disable_apache_site $name
 }

@@ -43,6 +43,13 @@ GITDIR["python-cinderclient"]=$DEST/python-cinderclient
 GITDIR["python-brick-cinderclient-ext"]=$DEST/python-brick-cinderclient-ext
 CINDER_DIR=$DEST/cinder

+if [[ $SERVICE_IP_VERSION == 6 ]]; then
+    CINDER_MY_IP="$HOST_IPV6"
+else
+    CINDER_MY_IP="$HOST_IP"
+fi
+
+
 # Cinder virtual environment
 if [[ ${USE_VENV} = True ]]; then
    PROJECT_VENV["cinder"]=${CINDER_DIR}.venv
@@ -69,6 +76,11 @@ CINDER_SERVICE_PORT_INT=${CINDER_SERVICE_PORT_INT:-18776}
 CINDER_SERVICE_PROTOCOL=${CINDER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 CINDER_SERVICE_LISTEN_ADDRESS=${CINDER_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)}

+# We do not need to report service status every 10s for devstack-like
+# deployments. In the gate this generates extra work for the services and the
+# database which are already taxed.
+CINDER_SERVICE_REPORT_INTERVAL=${CINDER_SERVICE_REPORT_INTERVAL:-120}
+
 # What type of LVM device should Cinder use for LVM backend
 # Defaults to auto, which will do thin provisioning if it's a fresh
 # volume group, otherwise it will do thick. The other valid choices are
@@ -76,6 +88,10 @@ CINDER_SERVICE_LISTEN_ADDRESS=${CINDER_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $S
 # thin provisioning.
 CINDER_LVM_TYPE=${CINDER_LVM_TYPE:-auto}

+# ``CINDER_USE_SERVICE_TOKEN`` is a mode where service token is passed along with
+# user token while communicating to external REST APIs like Glance.
+CINDER_USE_SERVICE_TOKEN=$(trueorfalse True CINDER_USE_SERVICE_TOKEN)
+
 # Default backends
 # The backend format is type:name where type is one of the supported backend
 # types (lvm, nfs, etc) and name is the identifier used in the Cinder
@@ -88,13 +104,33 @@ CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
 CINDER_VOLUME_CLEAR=${CINDER_VOLUME_CLEAR:-${CINDER_VOLUME_CLEAR_DEFAULT:-zero}}
 CINDER_VOLUME_CLEAR=$(echo ${CINDER_VOLUME_CLEAR} | tr '[:upper:]' '[:lower:]')

-# Default to lioadm
-CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-lioadm}
+VOLUME_TYPE_MULTIATTACH=${VOLUME_TYPE_MULTIATTACH:-multiattach}

-# EL and SUSE should only use lioadm
-if is_fedora || is_suse; then
-    if [[ ${CINDER_ISCSI_HELPER} != "lioadm" ]]; then
-        die "lioadm is the only valid Cinder target_helper config on this platform"
+if [[ -n "$CINDER_ISCSI_HELPER" ]]; then
+    if [[ -z "$CINDER_TARGET_HELPER" ]]; then
+        deprecated 'Using CINDER_ISCSI_HELPER is deprecated, use CINDER_TARGET_HELPER instead'
+        CINDER_TARGET_HELPER="$CINDER_ISCSI_HELPER"
+    else
+        deprecated 'Deprecated CINDER_ISCSI_HELPER is set, but is being overwritten by CINDER_TARGET_HELPER'
+    fi
+fi
+CINDER_TARGET_HELPER=${CINDER_TARGET_HELPER:-lioadm}
+
+if [[ $CINDER_TARGET_HELPER == 'nvmet' ]]; then
+    CINDER_TARGET_PROTOCOL=${CINDER_TARGET_PROTOCOL:-'nvmet_rdma'}
+    CINDER_TARGET_PREFIX=${CINDER_TARGET_PREFIX:-'nvme-subsystem-1'}
+    CINDER_TARGET_PORT=${CINDER_TARGET_PORT:-4420}
+else
+    CINDER_TARGET_PROTOCOL=${CINDER_TARGET_PROTOCOL:-'iscsi'}
+    CINDER_TARGET_PREFIX=${CINDER_TARGET_PREFIX:-'iqn.2010-10.org.openstack:'}
+    CINDER_TARGET_PORT=${CINDER_TARGET_PORT:-3260}
+fi
+
+
+# EL should only use lioadm
+if is_fedora; then
+    if [[ ${CINDER_TARGET_HELPER} != "lioadm" && ${CINDER_TARGET_HELPER} != 'nvmet' ]]; then
+        die "lioadm and nvmet are the only valid Cinder target_helper config on this platform"
    fi
 fi

@@ -151,6 +187,12 @@ fi
 # Environment variables to configure the image-volume cache
 CINDER_IMG_CACHE_ENABLED=${CINDER_IMG_CACHE_ENABLED:-True}

+# Environment variables to configure the optimized volume upload
+CINDER_UPLOAD_OPTIMIZED=${CINDER_UPLOAD_OPTIMIZED:-False}
+
+# Environment variables to configure the internal tenant during optimized volume upload
+CINDER_UPLOAD_INTERNAL_TENANT=${CINDER_UPLOAD_INTERNAL_TENANT:-False}
+
 # For limits, if left unset, it will use cinder defaults of 0 for unlimited
 CINDER_IMG_CACHE_SIZE_GB=${CINDER_IMG_CACHE_SIZE_GB:-}
 CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
@@ -160,6 +202,11 @@ CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
 # enable the cache for all cinder backends.
 CINDER_CACHE_ENABLED_FOR_BACKENDS=${CINDER_CACHE_ENABLED_FOR_BACKENDS:-$CINDER_ENABLED_BACKENDS}

+# Configure which cinder backends will have optimized volume upload, this takes the same
+# form as the CINDER_ENABLED_BACKENDS config option. By default it will
+# enable the cache for all cinder backends.
+CINDER_UPLOAD_OPTIMIZED_BACKENDS=${CINDER_UPLOAD_OPTIMIZED_BACKENDS:-$CINDER_ENABLED_BACKENDS}
+
 # Flag to set the oslo_policy.enforce_scope. This is used to switch
 # the  Volume API policies to start checking the scope of token. by default,
 # this flag is False.
@@ -187,7 +234,7 @@ function _cinder_cleanup_apache_wsgi {
 function cleanup_cinder {
    # ensure the volume group is cleared up because fails might
    # leave dead volumes in the group
-    if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
+    if [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
        local targets
        targets=$(sudo tgtadm --op show --mode target)
        if [ $? -ne 0 ]; then
@@ -215,8 +262,14 @@ function cleanup_cinder {
        else
            stop_service tgtd
        fi
-    else
+    elif [ "$CINDER_TARGET_HELPER" = "lioadm" ]; then
        sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
+    elif [ "$CINDER_TARGET_HELPER" = "nvmet" ]; then
+        # If we don't disconnect everything vgremove will block
+        sudo nvme disconnect-all
+        sudo nvmetcli clear
+    else
+        die $LINENO "Unknown value \"$CINDER_TARGET_HELPER\" for CINDER_TARGET_HELPER"
    fi

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
@@ -237,7 +290,7 @@ function cleanup_cinder {
    fi

    stop_process "c-api"
-    remove_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI"
+    remove_uwsgi_config "$CINDER_UWSGI_CONF" "cinder-wsgi"
 }

 # configure_cinder() - Set config files, create data dirs, etc
@@ -267,7 +320,7 @@ function configure_cinder {

    iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

-    iniset $CINDER_CONF DEFAULT target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF DEFAULT target_helper "$CINDER_TARGET_HELPER"
    iniset $CINDER_CONF database connection `database_connection_url cinder`
    iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
    iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"
@@ -275,11 +328,7 @@ function configure_cinder {
    iniset $CINDER_CONF DEFAULT osapi_volume_listen $CINDER_SERVICE_LISTEN_ADDRESS
    iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH
    iniset $CINDER_CONF oslo_concurrency lock_path $CINDER_STATE_PATH
-    if [[ $SERVICE_IP_VERSION == 6 ]]; then
-        iniset $CINDER_CONF DEFAULT my_ip "$HOST_IPV6"
-    else
-        iniset $CINDER_CONF DEFAULT my_ip "$HOST_IP"
-    fi
+    iniset $CINDER_CONF DEFAULT my_ip "$CINDER_MY_IP"
    iniset $CINDER_CONF key_manager backend cinder.keymgr.conf_key_mgr.ConfKeyManager
    iniset $CINDER_CONF key_manager fixed_key $(openssl rand -hex 16)
    if [[ -n "$CINDER_ALLOWED_DIRECT_URL_SCHEMES" ]]; then
@@ -296,6 +345,9 @@ function configure_cinder {
    # details and example failures.
    iniset $CINDER_CONF DEFAULT rpc_response_timeout 120

+    iniset $CINDER_CONF DEFAULT report_interval $CINDER_SERVICE_REPORT_INTERVAL
+    iniset $CINDER_CONF DEFAULT service_down_time $(($CINDER_SERVICE_REPORT_INTERVAL * 6))
+
    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local enabled_backends=""
        local default_name=""
@@ -316,6 +368,14 @@ function configure_cinder {
            iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}
        fi
        configure_cinder_image_volume_cache
+
+        # The upload optimization uses Cinder's clone volume functionality to
+        # clone the Image-Volume from source volume hence can only be
+        # performed when glance is using cinder as it's backend.
+        if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then
+            # Configure optimized volume upload
+            configure_cinder_volume_upload
+        fi
    fi

    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
@@ -377,54 +437,58 @@ function configure_cinder {
    if [[ ! -z "$CINDER_COORDINATION_URL" ]]; then
        iniset $CINDER_CONF coordination backend_url "$CINDER_COORDINATION_URL"
    elif is_service_enabled etcd3; then
-        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT"
+        # NOTE(jan.gutter): api_version can revert to default once tooz is
+        # updated with the etcd v3.4 defaults
+        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT?api_version=v3"
    fi

-    if [[ "$CINDER_ENFORCE_SCOPE" == True ]] ; then
+    if [[ "$CINDER_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
        iniset $CINDER_CONF oslo_policy enforce_scope true
        iniset $CINDER_CONF oslo_policy enforce_new_defaults true
+    else
+        iniset $CINDER_CONF oslo_policy enforce_scope false
+        iniset $CINDER_CONF oslo_policy enforce_new_defaults false
+    fi
+
+    if [ "$CINDER_USE_SERVICE_TOKEN" == "True" ]; then
+        init_cinder_service_user_conf
    fi
 }

 # create_cinder_accounts() - Set up common required cinder accounts

-# Tenant               User       Roles
+# Project              User       Roles
 # ------------------------------------------------------------------
-# service              cinder     admin        # if enabled
+# SERVICE_PROJECT_NAME cinder     service
+# SERVICE_PROJECT_NAME cinder     creator (if Barbican is enabled)

 # Migrated from keystone_data.sh
 function create_cinder_accounts {
    # Cinder
    if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

-        create_service_user "cinder"
+        local extra_role=""
+
+        # cinder needs the "creator" role in order to interact with barbican
+        if is_service_enabled barbican; then
+            extra_role=$(get_or_create_role "creator")
+        fi
+
+        create_service_user "cinder" $extra_role
+
+        local cinder_api_url
+        if [[ "$CINDER_USE_MOD_WSGI" == "False" ]]; then
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT"
+        else
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume"
+        fi

        # block-storage is the official service type
        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
-        if [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-        else
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-        fi
-
+        get_or_create_endpoint \
+            "block-storage" \
+            "$REGION_NAME" \
+            "$cinder_api_url/v3"
        configure_cinder_internal_tenant
    fi
 }
@@ -465,9 +529,9 @@ function init_cinder {
 function install_cinder {
    git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH
    setup_develop $CINDER_DIR
-    if [[ "$CINDER_ISCSI_HELPER" == "tgtadm" ]]; then
+    if [[ "$CINDER_TARGET_HELPER" == "tgtadm" ]]; then
        install_package tgt
-    elif [[ "$CINDER_ISCSI_HELPER" == "lioadm" ]]; then
+    elif [[ "$CINDER_TARGET_HELPER" == "lioadm" ]]; then
        if is_ubuntu; then
            # TODO(frickler): Workaround for https://launchpad.net/bugs/1819819
            sudo mkdir -p /etc/target
@@ -476,6 +540,43 @@ function install_cinder {
        else
            install_package targetcli
        fi
+    elif [[ "$CINDER_TARGET_HELPER" == "nvmet" ]]; then
+        install_package nvme-cli
+
+        # TODO: Remove manual installation of the dependency when the
+        # requirement is added to nvmetcli:
+        # http://lists.infradead.org/pipermail/linux-nvme/2022-July/033576.html
+        if is_ubuntu; then
+            install_package python3-configshell-fb
+        else
+            install_package python3-configshell
+        fi
+        # Install from source because Ubuntu doesn't have the package and some packaged versions didn't work on Python 3
+        pip_install git+git://git.infradead.org/users/hch/nvmetcli.git
+
+        sudo modprobe nvmet
+        sudo modprobe nvme-fabrics
+
+        if [[ $CINDER_TARGET_PROTOCOL == 'nvmet_rdma' ]]; then
+            install_package rdma-core
+            sudo modprobe nvme-rdma
+
+            # Create the Soft-RoCE device over the networking interface
+            local iface=${HOST_IP_IFACE:-`ip -br -$SERVICE_IP_VERSION a | grep $CINDER_MY_IP | awk '{print $1}'`}
+            if [[ -z "$iface" ]]; then
+                die $LINENO "Cannot find interface to bind Soft-RoCE"
+            fi
+
+            if ! sudo rdma link | grep $iface ; then
+                sudo rdma link add rxe_$iface type rxe netdev $iface
+            fi
+
+        elif [[ $CINDER_TARGET_PROTOCOL == 'nvmet_tcp' ]]; then
+            sudo modprobe nvme-tcp
+
+        else  # 'nvmet_fc'
+            sudo modprobe nvme-fc
+        fi
    fi
 }

@@ -512,18 +613,13 @@ function start_cinder {
        service_port=$CINDER_SERVICE_PORT_INT
        service_protocol="http"
    fi
-    if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
+    if [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
        if is_service_enabled c-vol; then
            # Delete any old stack.conf
            sudo rm -f /etc/tgt/conf.d/stack.conf
            _configure_tgt_for_config_d
            if is_ubuntu; then
                sudo service tgt restart
-            elif is_suse; then
-                # NOTE(dmllr): workaround restart bug
-                # https://bugzilla.suse.com/show_bug.cgi?id=934642
-                stop_service tgtd
-                start_service tgtd
            else
                restart_service tgtd
            fi
@@ -552,8 +648,13 @@ function start_cinder {
    fi

    run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
-    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
-    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
+    # Tune glibc for Python Services using single malloc arena for all threads
+    # and disabling dynamic thresholds to reduce memory usage when using native
+    # threads directly or via eventlet.tpool
+    # https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html
+    malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144"
+    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning"
+    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning"

    # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
    # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
@@ -568,6 +669,23 @@ function stop_cinder {
    stop_process c-vol
 }

+function create_one_type {
+    type_name=$1
+    property_key=$2
+    property_value=$3
+    # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
+    if is_service_enabled keystone; then
+        openstack --os-region-name="$REGION_NAME" volume type create --property $property_key="$property_value" $type_name
+    else
+        # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
+        # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
+        local cinder_url
+        cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create $type_name
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key $type_name set $property_key="$property_value"
+    fi
+}
+
 # create_volume_types() - Create Cinder's configured volume types
 function create_volume_types {
    # Create volume types
@@ -575,19 +693,13 @@ function create_volume_types {
        local be be_name
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_name=${be##*:}
-            # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
-            if is_service_enabled keystone; then
-                openstack --os-region-name="$REGION_NAME" volume type create --property volume_backend_name="${be_name}" ${be_name}
-            else
-                # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
-                # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
-                local cinder_url
-                cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create ${be_name}
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key ${be_name} set volume_backend_name=${be_name}
-            fi
+            create_one_type $be_name "volume_backend_name" $be_name
        done

+        if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then
+            create_one_type $VOLUME_TYPE_MULTIATTACH $VOLUME_TYPE_MULTIATTACH "<is> True"
+        fi
+
        # Increase quota for the service project if glance is using cinder,
        # since it's likely to occasionally go above the default 10 in parallel
        # test execution.
@@ -631,6 +743,24 @@ function configure_cinder_image_volume_cache {
    done
 }

+function configure_cinder_volume_upload {
+    # Expect UPLOAD_VOLUME_OPTIMIZED_FOR_BACKENDS to be a list of backends
+    # similar to CINDER_ENABLED_BACKENDS with NAME:TYPE where NAME will
+    # be the backend specific configuration stanza in cinder.conf.
+    local be be_name
+    for be in ${CINDER_UPLOAD_OPTIMIZED_BACKENDS//,/ }; do
+        be_name=${be##*:}
+
+        iniset $CINDER_CONF $be_name image_upload_use_cinder_backend $CINDER_UPLOAD_OPTIMIZED
+        iniset $CINDER_CONF $be_name image_upload_use_internal_tenant $CINDER_UPLOAD_INTERNAL_TENANT
+    done
+}
+
+function init_cinder_service_user_conf {
+    configure_keystone_authtoken_middleware $CINDER_CONF cinder service_user
+    iniset $CINDER_CONF service_user send_service_user_token True
+    iniset $CINDER_CONF service_user auth_strategy keystone
+}

 # Restore xtrace
 $_XTRACE_CINDER
@@ -50,7 +50,7 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.tests.fake_driver.FakeGateDriver"
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
-    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF $be_name target_helper "$CINDER_TARGET_HELPER"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"

    if [[ "$CINDER_VOLUME_CLEAR" == "non" ]]; then
@@ -50,7 +50,10 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMVolumeDriver"
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
-    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF $be_name target_helper "$CINDER_TARGET_HELPER"
+    iniset $CINDER_CONF $be_name target_protocol "$CINDER_TARGET_PROTOCOL"
+    iniset $CINDER_CONF $be_name target_port "$CINDER_TARGET_PORT"
+    iniset $CINDER_CONF $be_name target_prefix "$CINDER_TARGET_PREFIX"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"
    iniset $CINDER_CONF $be_name volume_clear "$CINDER_VOLUME_CLEAR"
 }
@@ -32,6 +32,15 @@ function configure_cinder_backend_nfs {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.nfs.NfsDriver"
    iniset $CINDER_CONF $be_name nfs_shares_config "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
+    iniset $CINDER_CONF $be_name nas_host localhost
+    iniset $CINDER_CONF $be_name nas_share_path ${NFS_EXPORT_DIR}
+    iniset $CINDER_CONF $be_name nas_secure_file_operations \
+        ${NFS_SECURE_FILE_OPERATIONS}
+    iniset $CINDER_CONF $be_name nas_secure_file_permissions \
+        ${NFS_SECURE_FILE_PERMISSIONS}
+
+    # NFS snapshot support is currently opt-in only.
+    iniset $CINDER_CONF $be_name nfs_snapshot_support True

    echo "$CINDER_NFS_SERVERPATH" | tee "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
 }
@@ -26,12 +26,15 @@ CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}


 function configure_cinder_backup_ceph {
-    sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
-    if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
-        sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+    # Execute this part only when cephadm is not used
+    if [[ "$CEPHADM_DEPLOY" = "False" ]]; then
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
+        if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
+            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+        fi
+        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
    fi
-    sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-    sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring

    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
    iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
@@ -20,13 +20,7 @@ if [[ -z "$MYSQL_SERVICE_NAME" ]]; then
    MYSQL_SERVICE_NAME=mysql
    if is_fedora && ! is_oraclelinux; then
        MYSQL_SERVICE_NAME=mariadb
-    elif is_suse && systemctl list-unit-files | grep -q 'mariadb\.service'; then
-        # Older mariadb packages on SLES 12 provided mysql.service.  The
-        # newer ones on SLES 12 and 15 use mariadb.service; they also
-        # provide a mysql.service symlink for backwards-compatibility, but
-        # let's not rely on that.
-        MYSQL_SERVICE_NAME=mariadb
-    elif [[ "$DISTRO" == "bullseye" ]]; then
+    elif [[ "$DISTRO" =~ bookworm|bullseye ]]; then
        MYSQL_SERVICE_NAME=mariadb
    fi
 fi
@@ -54,7 +48,7 @@ function cleanup_database_mysql {
    elif is_oraclelinux; then
        uninstall_package mysql-community-server
        sudo rm -rf /var/lib/mysql
-    elif is_suse || is_fedora; then
+    elif is_fedora; then
        uninstall_package mariadb-server
        sudo rm -rf /var/lib/mysql
    else
@@ -69,12 +63,12 @@ function recreate_database_mysql {
 }

 function configure_database_mysql {
-    local my_conf mysql slow_log
+    local my_conf mysql slow_log my_client_conf
    echo_summary "Configuring and starting MySQL"

    if is_ubuntu; then
        my_conf=/etc/mysql/my.cnf
-    elif is_suse || is_oraclelinux; then
+    elif is_oraclelinux; then
        my_conf=/etc/my.cnf
    elif is_fedora; then
        my_conf=/etc/my.cnf
@@ -86,11 +80,20 @@ function configure_database_mysql {
        exit_distro_not_supported "mysql configuration"
    fi

+    # Set fips mode on
+    if is_ubuntu; then
+        if is_fips_enabled; then
+            my_client_conf=/etc/mysql/mysql.conf.d/mysql.cnf
+            iniset -sudo $my_client_conf mysql ssl-fips-mode "on"
+            iniset -sudo $my_conf mysqld ssl-fips-mode "on"
+        fi
+    fi
+
    # Change bind-address from localhost (127.0.0.1) to any (::)
    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"

    # (Re)Start mysql-server
-    if is_fedora || is_suse; then
+    if is_fedora; then
        # service is not started by default
        start_service $MYSQL_SERVICE_NAME
    elif is_ubuntu; then
@@ -100,8 +103,13 @@ function configure_database_mysql {

    # Set the root password - only works the first time. For Ubuntu, we already
    # did that with debconf before installing the package, but we still try,
-    # because the package might have been installed already.
-    sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+    # because the package might have been installed already. We don't do this
+    # for Ubuntu 22.04 (jammy) because the authorization model change in
+    # version 10.4 of mariadb. See
+    # https://mariadb.org/authentication-in-mariadb-10-4/
+    if ! (is_ubuntu && [[ "$DISTRO" == "jammy" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then
+        sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+    fi

    # In case of Mariadb, giving hostname in arguments causes permission
    # problems as it expects connection through socket
@@ -114,14 +122,23 @@ function configure_database_mysql {
    # In mariadb e.g. on Ubuntu socket plugin is used for authentication
    # as root so it works only as sudo. To restore old "mysql like" behaviour,
    # we need to change auth plugin for root user
-    if is_ubuntu && [[ "$DISTRO" != "bullseye" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
-        sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
-        sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+    # TODO(frickler): simplify this logic
+    if is_ubuntu && [[ ! "$DISTRO" =~ bookworm|bullseye ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
+        if [[ "$DISTRO" == "jammy" ]]; then
+            # For Ubuntu 22.04 (jammy) we follow the model outlined in
+            # https://mariadb.org/authentication-in-mariadb-10-4/
+            sudo mysql -e "ALTER USER $DATABASE_USER@localhost IDENTIFIED VIA mysql_native_password USING PASSWORD('$DATABASE_PASSWORD');"
+        else
+            sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
+            sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+        fi
+    fi
+    if ! (is_ubuntu && [[ "$DISTRO" == "jammy" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then
+        # Create DB user if it does not already exist
+        sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
+        # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
+        sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"
    fi
-    # Create DB user if it does not already exist
-    sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
-    # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
-    sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"

    # Now update ``my.cnf`` for some local needs and restart the mysql service

@@ -150,6 +167,29 @@ function configure_database_mysql {
        iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1
    fi

+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        echo "enabling MySQL performance counting"
+
+        # Install our sqlalchemy plugin
+        pip_install ${TOP_DIR}/tools/dbcounter
+
+        # Create our stats database for accounting
+        recreate_database stats
+        mysql -u $DATABASE_USER -p$DATABASE_PASSWORD -h $MYSQL_HOST -e \
+              "CREATE TABLE queries (db VARCHAR(32), op VARCHAR(32),
+                count INT, PRIMARY KEY (db, op)) ENGINE MEMORY" stats
+    fi
+
+    if [[ "$MYSQL_REDUCE_MEMORY" == "True" ]]; then
+        iniset -sudo $my_conf mysqld read_buffer_size 64K
+        iniset -sudo $my_conf mysqld innodb_buffer_pool_size 16M
+        iniset -sudo $my_conf mysqld thread_stack 192K
+        iniset -sudo $my_conf mysqld thread_cache_size 8
+        iniset -sudo $my_conf mysqld tmp_table_size 8M
+        iniset -sudo $my_conf mysqld sort_buffer_size 8M
+        iniset -sudo $my_conf mysqld max_allowed_packet 8M
+    fi
+
    restart_service $MYSQL_SERVICE_NAME
 }

@@ -186,9 +226,6 @@ EOF
        elif is_fedora; then
            install_package mariadb-server mariadb-devel mariadb
            sudo systemctl enable $MYSQL_SERVICE_NAME
-        elif is_suse; then
-            install_package mariadb-server
-            sudo systemctl enable $MYSQL_SERVICE_NAME
        elif is_ubuntu; then
            install_package $MYSQL_SERVICE_NAME-server
        else
@@ -209,7 +246,17 @@ function install_database_python_mysql {

 function database_connection_url_mysql {
    local db=$1
-    echo "$BASE_SQL_CONN/$db?charset=utf8"
+    local plugin
+
+    # NOTE(danms): We don't enable perf on subnodes yet because the
+    # plugin is not installed there
+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        if is_service_enabled mysql; then
+            plugin="&plugin=dbcounter"
+        fi
+    fi
+
+    echo "$BASE_SQL_CONN/$db?charset=utf8$plugin"
 }


@@ -32,7 +32,7 @@ function cleanup_database_postgresql {
        # Get ruthless with mysql
        apt_get purge -y postgresql*
        return
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        uninstall_package postgresql-server
    else
        return
@@ -46,6 +46,10 @@ function recreate_database_postgresql {
    createdb -h $DATABASE_HOST -U$DATABASE_USER -l C -T template0 -E utf8 $db
 }

+function _exit_pg_init {
+    sudo cat /var/lib/pgsql/initdb_postgresql.log
+}
+
 function configure_database_postgresql {
    local pg_conf pg_dir pg_hba check_role version
    echo_summary "Configuring and starting PostgreSQL"
@@ -53,7 +57,9 @@ function configure_database_postgresql {
        pg_hba=/var/lib/pgsql/data/pg_hba.conf
        pg_conf=/var/lib/pgsql/data/postgresql.conf
        if ! sudo [ -e $pg_hba ]; then
+            trap _exit_pg_init EXIT
            sudo postgresql-setup initdb
+            trap - EXIT
        fi
    elif is_ubuntu; then
        version=`psql --version | cut -d ' ' -f3 | cut -d. -f1-2`
@@ -66,11 +72,6 @@ function configure_database_postgresql {
        pg_dir=`find /etc/postgresql -name pg_hba.conf|xargs dirname`
        pg_hba=$pg_dir/pg_hba.conf
        pg_conf=$pg_dir/postgresql.conf
-    elif is_suse; then
-        pg_hba=/var/lib/pgsql/data/pg_hba.conf
-        pg_conf=/var/lib/pgsql/data/postgresql.conf
-        # initdb is called when postgresql is first started
-        sudo [ -e $pg_hba ] || start_service postgresql
    else
        exit_distro_not_supported "postgresql configuration"
    fi
@@ -107,7 +108,7 @@ EOF
    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
        if is_ubuntu; then
            install_package postgresql
-        elif is_fedora || is_suse; then
+        elif is_fedora; then
            install_package postgresql-server
            if is_fedora; then
                sudo systemctl enable postgresql
@@ -40,12 +40,18 @@ function start_dstat {
    if is_service_enabled peakmem_tracker; then
        die $LINENO "The peakmem_tracker service has been removed, use memory_tracker instead"
    fi
+
+    # To enable file_tracker add:
+    #    enable_service file_tracker
+    # to your localrc
+    run_process file_tracker "$TOP_DIR/tools/file_tracker.sh"
 }

 # stop_dstat() stop dstat process
 function stop_dstat {
    stop_process dstat
    stop_process memory_tracker
+    stop_process file_tracker
 }

 # Restore xtrace
@@ -51,7 +51,7 @@ function start_etcd3 {
    fi
    cmd+=" --listen-client-urls http://$SERVICE_HOST:$ETCD_PORT"
    if [ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]; then
-        cmd+=" --debug"
+        cmd+=" --log-level=debug"
    fi

    local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
@@ -47,9 +47,8 @@ USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE)
 # from CINDER_ENABLED_BACKENDS
 GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
 GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
-# NOTE (abhishekk): For opensuse data files are stored in different directory
-if is_opensuse; then
-    GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/etc/glance
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+    GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance
 fi
 # When Cinder is used as a glance store, you can optionally configure cinder to
 # optimize bootable volume creation by allowing volumes to be cloned directly
@@ -76,6 +75,7 @@ GLANCE_MULTIPLE_FILE_STORES=${GLANCE_MULTIPLE_FILE_STORES:-fast}
 GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:-fast}

 GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache}
+GLANCE_CACHE_DRIVER=${GLANCE_CACHE_DRIVER:-centralized_db}

 # Full Glance functionality requires running in standalone mode. If we are
 # not in uwsgi mode, then we are standalone, otherwise allow separate control.
@@ -99,10 +99,13 @@ GLANCE_USE_IMPORT_WORKFLOW=$(trueorfalse False GLANCE_USE_IMPORT_WORKFLOW)
 GLANCE_ENABLE_QUOTAS=$(trueorfalse True GLANCE_ENABLE_QUOTAS)

 # Flag to set the oslo_policy.enforce_scope. This is used to switch
-# the Image API policies to start checking the scope of token. By Default,
-# this flag is False.
+# This is used to disable the Image API policies scope and new defaults.
+# By Default, it is True.
 # For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
-GLANCE_ENFORCE_SCOPE=$(trueorfalse False GLANCE_ENFORCE_SCOPE)
+GLANCE_ENFORCE_SCOPE=$(trueorfalse True GLANCE_ENFORCE_SCOPE)
+
+# Flag to disable image format inspection on upload
+GLANCE_ENFORCE_IMAGE_FORMAT=$(trueorfalse True GLANCE_ENFORCE_IMAGE_FORMAT)

 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
@@ -168,6 +171,7 @@ function cleanup_glance {
        # Cleanup reserved stores directories
        sudo rm -rf $GLANCE_STAGING_DIR $GLANCE_TASKS_DIR
    fi
+    remove_uwsgi_config "$GLANCE_UWSGI_CONF" "glance-wsgi-api"
 }

 # Set multiple cinder store related config options for each of the cinder store
@@ -330,6 +334,7 @@ function configure_glance {
    iniset $GLANCE_API_CONF database connection $dburl
    iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_API_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_API_CONF oslo_concurrency lock_path $GLANCE_LOCK_DIR
    iniset $GLANCE_API_CONF paste_deploy flavor keystone+cachemanagement
    configure_keystone_authtoken_middleware $GLANCE_API_CONF glance
@@ -341,6 +346,7 @@ function configure_glance {
    # Only use these if you know what you are doing!  See OSSN-0065
    iniset $GLANCE_API_CONF DEFAULT show_image_direct_url $GLANCE_SHOW_DIRECT_URL
    iniset $GLANCE_API_CONF DEFAULT show_multiple_locations $GLANCE_SHOW_MULTIPLE_LOCATIONS
+    iniset $GLANCE_API_CONF image_format require_image_format_match $GLANCE_ENFORCE_IMAGE_FORMAT

    # Configure glance_store
    configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES
@@ -393,6 +399,7 @@ function configure_glance {
    iniset $GLANCE_CACHE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $GLANCE_CACHE_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_SERVICE_URI
    iniset $GLANCE_CACHE_CONF DEFAULT admin_tenant_name $SERVICE_PROJECT_NAME
    iniset $GLANCE_CACHE_CONF DEFAULT admin_user glance
@@ -430,12 +437,17 @@ function configure_glance {
        iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
        iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
        iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
+        iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url $GLANCE_URL
    fi

-    if [[ "$GLANCE_ENFORCE_SCOPE" == True ]] ; then
+    if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
        iniset $GLANCE_API_CONF oslo_policy enforce_scope true
        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults true
        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac true
+    else
+        iniset $GLANCE_API_CONF oslo_policy enforce_scope false
+        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults false
+        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac false
    fi
 }

@@ -540,7 +552,7 @@ function glance_remote_conf {
 # start_glance_remote_clone() - Clone the regular glance api worker
 function start_glance_remote_clone {
    local glance_remote_conf_dir glance_remote_port remote_data
-    local glance_remote_uwsgi
+    local glance_remote_uwsgi venv

    glance_remote_conf_dir="$(glance_remote_conf "")"
    glance_remote_port=$(get_random_port)
@@ -578,12 +590,16 @@ function start_glance_remote_clone {
    # We need to create the systemd service for the clone, but then
    # change it to include an Environment line to point the WSGI app
    # at the alternate config directory.
+    if [[ "$GLOBAL_VENV" == True ]]; then
+        venv="--venv $DEVSTACK_VENV"
+    fi
    write_uwsgi_user_unit_file devstack@g-api-r.service "$(which uwsgi) \
                               --procname-prefix \
                               glance-api-remote \
-                               --ini $glance_remote_uwsgi" \
+                               --ini $glance_remote_uwsgi \
+                               $venv" \
                               "" "$STACK_USER"
-    iniset -sudo ${SYSTEMD_DIR}/devstack@g-api-r.service \
+    iniadd -sudo ${SYSTEMD_DIR}/devstack@g-api-r.service \
           "Service" "Environment" \
           "OS_GLANCE_CONFIG_DIR=$glance_remote_conf_dir"

@@ -109,12 +109,21 @@ function configure_horizon {
        _horizon_config_set $local_settings "" OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT "True"
    fi

+    if is_service_enabled c-bak; then
+        _horizon_config_set $local_settings OPENSTACK_CINDER_FEATURES enable_backup "True"
+    fi
+
    # Create an empty directory that apache uses as docroot
    sudo mkdir -p $HORIZON_DIR/.blackhole

    local horizon_conf
    horizon_conf=$(apache_site_config_for horizon)

+    local wsgi_venv_config=""
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV"
+    fi
+
    # Configure apache to run horizon
    # Set up the django horizon application to serve via apache/wsgi
    sudo sh -c "sed -e \"
@@ -124,12 +133,13 @@ function configure_horizon {
        s,%APACHE_NAME%,$APACHE_NAME,g;
        s,%DEST%,$DEST,g;
        s,%WEBROOT%,$HORIZON_APACHE_ROOT,g;
+        s,%WSGIPYTHONHOME%,$wsgi_venv_config,g;
    \" $FILES/apache-horizon.template >$horizon_conf"

    if is_ubuntu; then
        disable_apache_site 000-default
        sudo touch $horizon_conf
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        : # nothing to do
    else
        exit_distro_not_supported "horizon apache configuration"
@@ -163,6 +173,10 @@ function install_horizon {
    # Apache installation, because we mark it NOPRIME
    install_apache_wsgi

+    # Install the memcache library so that horizon can use memcached as its
+    # cache backend
+    pip_install_gr pymemcache
+
    git_clone $HORIZON_REPO $HORIZON_DIR $HORIZON_BRANCH
 }

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Kernel Samepage Merging (KSM)
+# -----------------------------
+
+# Processes that mark their memory as mergeable can share identical memory
+# pages if KSM is enabled. This is particularly useful for nova + libvirt
+# backends but any other setup that marks its memory as mergeable can take
+# advantage. The drawback is there is higher cpu load; however, we tend to
+# be memory bound not cpu bound so enable KSM by default but allow people
+# to opt out if the CPU time is more important to them.
+ENABLE_KSM=$(trueorfalse True ENABLE_KSM)
+ENABLE_KSMTUNED=$(trueorfalse True ENABLE_KSMTUNED)
+function configure_ksm {
+    if [[ $ENABLE_KSMTUNED == "True" ]] ; then
+        install_package "ksmtuned"
+    fi
+    if [[ -f /sys/kernel/mm/ksm/run ]] ; then
+        echo $(bool_to_int ENABLE_KSM) | sudo tee /sys/kernel/mm/ksm/run
+    fi
+}
+
+# Compressed swap (ZSWAP)
+#------------------------
+
+# as noted in the kernel docs https://docs.kernel.org/admin-guide/mm/zswap.html
+# Zswap is a lightweight compressed cache for swap pages.
+# It takes pages that are in the process of being swapped out and attempts
+# to compress them into a dynamically allocated RAM-based memory pool.
+# zswap basically trades CPU cycles for potentially reduced swap I/O.
+# This trade-off can also result in a significant performance improvement
+# if reads from the compressed cache are faster than reads from a swap device.
+
+ENABLE_ZSWAP=$(trueorfalse False ENABLE_ZSWAP)
+# lz4 is very fast although it does not have the best compression
+# zstd has much better compression but more latency
+ZSWAP_COMPRESSOR=${ZSWAP_COMPRESSOR:="lz4"}
+ZSWAP_ZPOOL=${ZSWAP_ZPOOL:="z3fold"}
+function configure_zswap {
+    if [[ $ENABLE_ZSWAP == "True" ]] ; then
+        # Centos 9 stream seems to only support enabling but not run time
+        # tuning so dont try to choose better default on centos
+        if is_ubuntu; then
+            echo ${ZSWAP_COMPRESSOR} | sudo tee /sys/module/zswap/parameters/compressor
+            echo ${ZSWAP_ZPOOL} | sudo tee /sys/module/zswap/parameters/zpool
+        fi
+        echo 1 | sudo tee /sys/module/zswap/parameters/enabled
+        # print curent zswap kernel config
+        sudo grep -R . /sys/module/zswap/parameters || /bin/true
+    fi
+}
+
+ENABLE_SYSCTL_MEM_TUNING=$(trueorfalse False ENABLE_SYSCTL_MEM_TUNING)
+function configure_sysctl_mem_parmaters {
+    if [[ $ENABLE_SYSCTL_MEM_TUNING == "True" ]] ; then
+        # defer write when memory is available
+        sudo sysctl -w vm.dirty_ratio=60
+        sudo sysctl -w vm.dirty_background_ratio=10
+        sudo sysctl -w vm.vfs_cache_pressure=50
+        # assume swap is compressed so on new kernels
+        # give it equal priority as page cache which is
+        # uncompressed. on kernels < 5.8 the max is 100
+        # not 200 so it will strongly prefer swapping.
+        sudo sysctl -w vm.swappiness=100
+        sudo grep -R . /proc/sys/vm/  || /bin/true
+    fi
+}
+
+function configure_host_mem {
+    configure_zswap
+    configure_ksm
+    configure_sysctl_mem_parmaters
+}
+
+ENABLE_SYSCTL_NET_TUNING=$(trueorfalse False ENABLE_SYSCTL_NET_TUNING)
+function configure_sysctl_net_parmaters {
+    if [[ $ENABLE_SYSCTL_NET_TUNING == "True" ]] ; then
+        # detect dead TCP connections after 120 seconds
+        sudo sysctl -w net.ipv4.tcp_keepalive_time=60
+        sudo sysctl -w net.ipv4.tcp_keepalive_intvl=10
+        sudo sysctl -w net.ipv4.tcp_keepalive_probes=6
+        # reudce network latency for new connections
+        sudo sysctl -w net.ipv4.tcp_fastopen=3
+        # print tcp options
+        sudo grep -R . /proc/sys/net/ipv4/tcp* || /bin/true
+        # disable qos by default
+        sudo sysctl -w net.core.default_qdisc=pfifo_fast
+    fi
+}
+
+function configure_host_net {
+    configure_sysctl_net_parmaters
+}
+
+function tune_host {
+    configure_host_mem
+    configure_host_net
+}
@@ -31,7 +31,7 @@ function install_infra {
    local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv"
    [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV
    # We don't care about testing git pbr in the requirements venv.
-    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr
+    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr setuptools
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR

    # Unset the PIP_VIRTUAL_ENV so that PBR does not end up trapped
@@ -150,7 +150,7 @@ function cleanup_keystone {
        sudo rm -f $(apache_site_config_for keystone)
    else
        stop_process "keystone"
-        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI"
+        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "keystone-wsgi-public"
        sudo rm -f $(apache_site_config_for keystone-wsgi-public)
    fi
 }
@@ -265,10 +265,15 @@ function configure_keystone {
        iniset $KEYSTONE_CONF security_compliance lockout_duration $KEYSTONE_LOCKOUT_DURATION
        iniset $KEYSTONE_CONF security_compliance unique_last_password_count $KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT
    fi
-    if [[ "$KEYSTONE_ENFORCE_SCOPE" == True ]] ; then
+
+    iniset $KEYSTONE_CONF oslo_policy policy_file policy.yaml
+
+    if [[ "$KEYSTONE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
        iniset $KEYSTONE_CONF oslo_policy enforce_scope true
        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults true
-        iniset $KEYSTONE_CONF oslo_policy policy_file policy.yaml
+    else
+        iniset $KEYSTONE_CONF oslo_policy enforce_scope false
+        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults false
    fi
 }

@@ -39,13 +39,6 @@ elif is_fedora; then
    LDAP_OLCDB_NUMBER=2
    LDAP_OLCDB_TYPE=hdb
    LDAP_ROOTPW_COMMAND=add
-elif is_suse; then
-    # SUSE has slappasswd in /usr/sbin/
-    PATH=$PATH:/usr/sbin/
-    LDAP_OLCDB_NUMBER=1
-    LDAP_OLCDB_TYPE=hdb
-    LDAP_ROOTPW_COMMAND=add
-    LDAP_SERVICE_NAME=ldap
 fi


@@ -76,8 +69,6 @@ function cleanup_ldap {
        sudo rm -rf /etc/ldap/ldap.conf /var/lib/ldap
    elif is_fedora; then
        sudo rm -rf /etc/openldap /var/lib/ldap
-    elif is_suse; then
-        sudo rm -rf /var/lib/ldap
    fi
 }

@@ -126,11 +117,6 @@ function install_ldap {
        configure_ldap
    elif is_fedora; then
        start_ldap
-    elif is_suse; then
-        _ldap_varsubst $FILES/ldap/suse-base-config.ldif.in >$tmp_ldap_dir/suse-base-config.ldif
-        sudo slapadd -F /etc/openldap/slapd.d/ -bcn=config -l $tmp_ldap_dir/suse-base-config.ldif
-        sudo sed -i '/^OPENLDAP_START_LDAPI=/s/"no"/"yes"/g' /etc/sysconfig/openldap
-        start_ldap
    fi

    echo "LDAP_PASSWORD is $LDAP_PASSWORD"
@@ -129,19 +129,25 @@ function init_lvm_volume_group {
    local vg=$1
    local size=$2

-    # Start the tgtd service on Fedora and SUSE if tgtadm is used
-    if  is_fedora || is_suse  && [[ "$CINDER_ISCSI_HELPER" = "tgtadm" ]]; then
+    # Start the tgtd service on Fedora if tgtadm is used
+    if  is_fedora; then
        start_service tgtd
    fi

    # Start with a clean volume group
    _create_lvm_volume_group $vg $size

-    # Remove iscsi targets
-    if [ "$CINDER_ISCSI_HELPER" = "lioadm" ]; then
-        sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
-    else
-        sudo tgtadm --op show --mode target | awk '/Target/ {print $3}' | sudo xargs -r -n1 tgt-admin --delete
+    if is_service_enabled cinder; then
+        # Remove iscsi targets
+        if [ "$CINDER_TARGET_HELPER" = "lioadm" ]; then
+            sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
+        elif [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
+            sudo tgtadm --op show --mode target | awk '/Target/ {print $3}' | sudo xargs -r -n1 tgt-admin --delete
+        elif [ "$CINDER_TARGET_HELPER" = "nvmet" ]; then
+            # If we don't disconnect everything vgremove will block
+            sudo nvme disconnect-all
+            sudo nvmetcli clear
+        fi
    fi
    _clean_lvm_volume_group $vg
 }
@@ -194,7 +200,7 @@ function set_lvm_filter {
    filter_string=$filter_string$filter_suffix

    clean_lvm_filter
-    sudo sed -i "/# global_filter = \[*\]/a\    $global_filter$filter_string" /etc/lvm/lvm.conf
+    sudo sed -i "/# global_filter = \[.*\]/a\        $filter_string" /etc/lvm/lvm.conf
    echo_summary "set lvm.conf device global_filter to: $filter_string"
 }

@@ -13,7 +13,7 @@ Plugin specific configuration variables should be in this file.

 functions
 ---------
-``lib/neutron-legacy`` calls the following functions when the ``$Q_PLUGIN`` is enabled
+``lib/neutron`` calls the following functions when the ``$Q_PLUGIN`` is enabled

 * ``neutron_plugin_create_nova_conf`` :
  optionally set options in nova_conf
@@ -67,7 +67,7 @@ function has_neutron_plugin_security_group {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -72,7 +72,7 @@ function has_neutron_plugin_security_group {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -97,7 +97,7 @@ function neutron_plugin_setup_interface_driver {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -67,7 +67,7 @@ function neutron_plugin_configure_common {
    Q_PLUGIN_CLASS="ml2"
    # The ML2 plugin delegates L3 routing/NAT functionality to
    # the L3 service plugin which must therefore be specified.
-    _neutron_service_plugin_class_add $ML2_L3_PLUGIN
+    neutron_service_plugin_class_add $ML2_L3_PLUGIN
 }

 function neutron_plugin_configure_service {
@@ -111,20 +111,13 @@ function neutron_plugin_configure_service {
            fi
        fi
    fi
-    # REVISIT(rkukura): Setting firewall_driver here for
-    # neutron.agent.securitygroups_rpc.is_firewall_enabled() which is
-    # used in the server, in case no L2 agent is configured on the
-    # server's node. If an L2 agent is configured, this will get
-    # overridden with the correct driver. The ml2 plugin should
-    # instead use its own config variable to indicate whether security
-    # groups is enabled, and that will need to be set here instead.
-    if [[ "$Q_USE_SECGROUP" == "True" ]]; then
-        iniset /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver neutron.agent.not.a.real.FirewallDriver
-    else
-        iniset /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver neutron.agent.firewall.NoopFirewallDriver
-    fi
+    populate_ml2_config /$Q_PLUGIN_CONF_FILE securitygroup enable_security_group=$Q_USE_SECGROUP

    populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 mechanism_drivers=$Q_ML2_PLUGIN_MECHANISM_DRIVERS
+    if [[ "$Q_ML2_PLUGIN_MECHANISM_DRIVERS" == *"linuxbridge"* ]]; then
+        iniset $NEUTRON_CONF experimental linuxbridge True
+    fi
+    populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 overlay_ip_version=$TUNNEL_IP_VERSION

    if [[ -n "$Q_ML2_PLUGIN_TYPE_DRIVERS" ]]; then
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 type_drivers=$Q_ML2_PLUGIN_TYPE_DRIVERS
@@ -68,7 +68,7 @@ function neutron_plugin_setup_interface_driver {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -91,16 +91,23 @@ OVN_GENEVE_OVERHEAD=${OVN_GENEVE_OVERHEAD:-38}
 # http://www.openvswitch.org/support/dist-docs/ovs-appctl.8.txt
 OVN_DBS_LOG_LEVEL=${OVN_DBS_LOG_LEVEL:-info}

+# OVN metadata agent configuration
 OVN_META_CONF=$NEUTRON_CONF_DIR/neutron_ovn_metadata_agent.ini
 OVN_META_DATA_HOST=${OVN_META_DATA_HOST:-$(ipv6_unquote $SERVICE_HOST)}

+# OVN agent configuration
+OVN_AGENT_CONF=$NEUTRON_CONF_DIR/plugins/ml2/ovn_agent.ini
+OVN_AGENT_EXTENSIONS=${OVN_AGENT_EXTENSIONS:-}
+
 # If True (default) the node will be considered a gateway node.
 ENABLE_CHASSIS_AS_GW=$(trueorfalse True ENABLE_CHASSIS_AS_GW)
 OVN_L3_CREATE_PUBLIC_NETWORK=$(trueorfalse True OVN_L3_CREATE_PUBLIC_NETWORK)

 export OVSDB_SERVER_LOCAL_HOST=$SERVICE_LOCAL_HOST
+TUNNEL_IP=$TUNNEL_ENDPOINT_IP
 if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
    OVSDB_SERVER_LOCAL_HOST=[$OVSDB_SERVER_LOCAL_HOST]
+    TUNNEL_IP=[$TUNNEL_IP]
 fi

 OVN_IGMP_SNOOPING_ENABLE=$(trueorfalse False OVN_IGMP_SNOOPING_ENABLE)
@@ -130,6 +137,7 @@ OVN_RUNDIR=$OVS_PREFIX/var/run/ovn

 NEUTRON_OVN_BIN_DIR=$(get_python_exec_prefix)
 NEUTRON_OVN_METADATA_BINARY="neutron-ovn-metadata-agent"
+NEUTRON_OVN_AGENT_BINARY="neutron-ovn-agent"

 STACK_GROUP="$( id --group --name "$STACK_USER" )"

@@ -169,12 +177,23 @@ Q_LOG_DRIVER_LOG_BASE=${Q_LOG_DRIVER_LOG_BASE:-acl_log_meter}
 # Utility Functions
 # -----------------

+function wait_for_db_file {
+    local count=0
+    while [ ! -f $1 ]; do
+        sleep 1
+        count=$((count+1))
+        if [ "$count" -gt 40 ]; then
+            die $LINENO "DB File $1 not found"
+        fi
+    done
+}
+
 function wait_for_sock_file {
    local count=0
    while [ ! -S $1 ]; do
        sleep 1
        count=$((count+1))
-        if [ "$count" -gt 5 ]; then
+        if [ "$count" -gt 40 ]; then
            die $LINENO "Socket $1 not found"
        fi
    done
@@ -231,11 +250,12 @@ function _run_process {
    local cmd="$2"
    local stop_cmd="$3"
    local group=$4
-    local user=${5:-$STACK_USER}
+    local user=$5
+    local rundir=${6:-$OVS_RUNDIR}

    local systemd_service="devstack@$service.service"
    local unit_file="$SYSTEMD_DIR/$systemd_service"
-    local environment="OVN_RUNDIR=$OVS_RUNDIR OVN_DBDIR=$OVN_DATADIR OVN_LOGDIR=$LOGDIR OVS_RUNDIR=$OVS_RUNDIR OVS_DBDIR=$OVS_DATADIR OVS_LOGDIR=$LOGDIR"
+    local environment="OVN_RUNDIR=$OVN_RUNDIR OVN_DBDIR=$OVN_DATADIR OVN_LOGDIR=$LOGDIR OVS_RUNDIR=$OVS_RUNDIR OVS_DBDIR=$OVS_DATADIR OVS_LOGDIR=$LOGDIR"

    echo "Starting $service executed command": $cmd

@@ -251,14 +271,14 @@ function _run_process {

    _start_process $systemd_service

-    local testcmd="test -e $OVS_RUNDIR/$service.pid"
+    local testcmd="test -e $rundir/$service.pid"
    test_with_retry "$testcmd" "$service did not start" $SERVICE_TIMEOUT 1
    local service_ctl_file
-    service_ctl_file=$(ls $OVS_RUNDIR | grep $service | grep ctl)
+    service_ctl_file=$(ls $rundir | grep $service | grep ctl)
    if [ -z "$service_ctl_file" ]; then
        die $LINENO "ctl file for service $service is not present."
    fi
-    sudo ovs-appctl -t $OVS_RUNDIR/$service_ctl_file vlog/set console:off syslog:info file:info
+    sudo ovs-appctl -t $rundir/$service_ctl_file vlog/set console:off syslog:info file:info
 }

 function clone_repository {
@@ -274,7 +294,7 @@ function clone_repository {
 function create_public_bridge {
    # Create the public bridge that OVN will use
    sudo ovs-vsctl --may-exist add-br $PUBLIC_BRIDGE -- set bridge $PUBLIC_BRIDGE protocols=OpenFlow13,OpenFlow15
-    sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=$PHYSICAL_NETWORK:$PUBLIC_BRIDGE
+    sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=${OVN_BRIDGE_MAPPINGS}
    _configure_public_network_connectivity
 }

@@ -320,8 +340,24 @@ function compile_ovn {
        ./boot.sh
    fi

+    # NOTE(mnaser): OVN requires that you build using the OVS from the
+    #               submodule.
+    #
+    #               https://github.com/ovn-org/ovn/blob/3fb397b63663297acbcbf794e1233951222ae5af/Documentation/intro/install/general.rst#bootstrapping
+    #               https://github.com/ovn-org/ovn/issues/128
+    git submodule update --init
+    pushd ovs
+    if [ ! -f configure ] ; then
+        ./boot.sh
+    fi
    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
-        ./configure --with-ovs-source=$DEST/$OVS_REPO_NAME $prefix $localstatedir
+        ./configure
+    fi
+    make -j$(($(nproc) + 1))
+    popd
+
+    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
+        ./configure $prefix $localstatedir
    fi
    make -j$(($(nproc) + 1))
    sudo make install
@@ -334,7 +370,7 @@ function compile_ovn {

 # OVN service sanity check
 function ovn_sanity_check {
-    if is_service_enabled q-agt neutron-agt; then
+    if is_service_enabled q-agt neutron-agent; then
        die $LINENO "The q-agt/neutron-agt service must be disabled with OVN."
    elif is_service_enabled q-l3 neutron-l3; then
        die $LINENO "The q-l3/neutron-l3 service must be disabled with OVN."
@@ -357,10 +393,6 @@ function install_ovn {

    sudo mkdir -p $OVS_RUNDIR
    sudo chown $(whoami) $OVS_RUNDIR
-    # NOTE(lucasagomes): To keep things simpler, let's reuse the same
-    # RUNDIR for both OVS and OVN. This way we avoid having to specify the
-    # --db option in the ovn-{n,s}bctl commands while playing with DevStack
-    sudo ln -s $OVS_RUNDIR $OVN_RUNDIR

    if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
        # If OVS is already installed, remove it, because we're about to
@@ -384,6 +416,8 @@ function install_ovn {
        sudo mkdir -p $OVS_PREFIX/var/log/ovn
        sudo chown $(whoami) $OVS_PREFIX/var/log/ovn
    else
+        # Load fixup_ovn_centos
+        source ${TOP_DIR}/tools/fixup_stuff.sh
        fixup_ovn_centos
        install_package $(get_packages openvswitch)
        install_package $(get_packages ovn)
@@ -449,7 +483,7 @@ function filter_network_api_extensions {
 function configure_ovn_plugin {
    echo "Configuring Neutron for OVN"

-    if is_service_enabled q-svc ; then
+    if is_service_enabled q-svc neutron-api; then
        filter_network_api_extensions
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2_type_geneve max_header_size=$OVN_GENEVE_OVERHEAD
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_connection="$OVN_NB_REMOTE"
@@ -473,7 +507,9 @@ function configure_ovn_plugin {
            inicomment /$Q_PLUGIN_CONF_FILE network_log local_output_log_base="$Q_LOG_DRIVER_LOG_BASE"
        fi

-        if is_service_enabled q-ovn-metadata-agent; then
+        if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
+        elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
        else
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=False
@@ -494,7 +530,9 @@ function configure_ovn_plugin {
    fi

    if is_service_enabled n-api-meta ; then
-        if is_service_enabled q-ovn-metadata-agent ; then
+        if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
+            iniset $NOVA_CONF neutron service_metadata_proxy True
+        elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then
            iniset $NOVA_CONF neutron service_metadata_proxy True
        fi
    fi
@@ -527,29 +565,42 @@ function configure_ovn {
    fi

    # Metadata
-    if is_service_enabled q-ovn-metadata-agent && is_service_enabled ovn-controller; then
+    local sample_file=""
+    local config_file=""
+    if is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]] && is_service_enabled ovn-controller; then
+        sample_file=$NEUTRON_DIR/etc/neutron/plugins/ml2/ovn_agent.ini.sample
+        config_file=$OVN_AGENT_CONF
+    elif is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent && is_service_enabled ovn-controller; then
+        sample_file=$NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample
+        config_file=$OVN_META_CONF
+    fi
+    if [ -n "$config_file" ]; then
        sudo install -d -o $STACK_USER $NEUTRON_CONF_DIR

        mkdir -p $NEUTRON_DIR/etc/neutron/plugins/ml2
        (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)

-        cp $NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample $OVN_META_CONF
-        configure_root_helper_options $OVN_META_CONF
+        cp $sample_file $config_file
+        configure_root_helper_options $config_file

-        iniset $OVN_META_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
-        iniset $OVN_META_CONF DEFAULT nova_metadata_host $OVN_META_DATA_HOST
-        iniset $OVN_META_CONF DEFAULT metadata_workers $API_WORKERS
-        iniset $OVN_META_CONF DEFAULT state_path $NEUTRON_STATE_PATH
-        iniset $OVN_META_CONF ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
-        iniset $OVN_META_CONF ovn ovn_sb_connection $OVN_SB_REMOTE
+        iniset $config_file DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
+        iniset $config_file DEFAULT nova_metadata_host $OVN_META_DATA_HOST
+        iniset $config_file DEFAULT metadata_workers $API_WORKERS
+        iniset $config_file DEFAULT state_path $DATA_DIR/neutron
+        iniset $config_file ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
+        iniset $config_file ovn ovn_sb_connection $OVN_SB_REMOTE
        if is_service_enabled tls-proxy; then
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_ca_cert $INT_CA_DIR/ca-chain.pem
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_certificate $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_private_key $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key
        fi
+        if [[ $config_file == $OVN_AGENT_CONF ]]; then
+            iniset $config_file agent extensions $OVN_AGENT_EXTENSIONS
+            iniset $config_file ovn ovn_nb_connection $OVN_NB_REMOTE
+        fi
    fi
 }

@@ -575,6 +626,7 @@ function init_ovn {
    rm -f $OVS_DATADIR/.*.db.~lock~
    sudo rm -f $OVN_DATADIR/*.db
    sudo rm -f $OVN_DATADIR/.*.db.~lock~
+    sudo rm -f $OVN_RUNDIR/*.sock
 }

 function _start_ovs {
@@ -601,12 +653,12 @@ function _start_ovs {
                dbcmd+=" --remote=db:hardware_vtep,Global,managers $OVS_DATADIR/vtep.db"
            fi
            dbcmd+=" $OVS_DATADIR/conf.db"
-            _run_process ovsdb-server "$dbcmd"
+            _run_process ovsdb-server "$dbcmd" "" "$STACK_GROUP" "root" "$OVS_RUNDIR"

            # Note: ovn-controller will create and configure br-int once it is started.
            # So, no need to create it now because nothing depends on that bridge here.
            local ovscmd="$OVS_SBINDIR/ovs-vswitchd --log-file --pidfile --detach"
-            _run_process ovs-vswitchd "$ovscmd" "" "$STACK_GROUP" "root"
+            _run_process ovs-vswitchd "$ovscmd" "" "$STACK_GROUP" "root" "$OVS_RUNDIR"
        else
            _start_process "$OVSDB_SERVER_SERVICE"
            _start_process "$OVS_VSWITCHD_SERVICE"
@@ -626,8 +678,8 @@ function _start_ovs {
        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-remote="$OVN_SB_REMOTE"
        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-bridge="br-int"
        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-type="geneve"
-        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$HOST_IP"
-        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:hostname="$LOCAL_HOSTNAME"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$TUNNEL_IP"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:hostname=$(hostname)
        # Select this chassis to host gateway routers
        if [[ "$ENABLE_CHASSIS_AS_GW" == "True" ]]; then
            sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-cms-options="enable-chassis-as-gw"
@@ -641,11 +693,11 @@ function _start_ovs {
        if is_service_enabled ovn-controller-vtep ; then
            ovn_base_setup_bridge br-v
            vtep-ctl add-ps br-v
-            vtep-ctl set Physical_Switch br-v tunnel_ips=$HOST_IP
+            vtep-ctl set Physical_Switch br-v tunnel_ips=$TUNNEL_IP

            enable_service ovs-vtep
            local vtepcmd="$OVS_SCRIPTDIR/ovs-vtep --log-file --pidfile --detach br-v"
-            _run_process ovs-vtep "$vtepcmd" "" "$STACK_GROUP" "root"
+            _run_process ovs-vtep "$vtepcmd" "" "$STACK_GROUP" "root" "$OVS_RUNDIR"

            vtep-ctl set-manager tcp:$HOST_IP:6640
        fi
@@ -668,9 +720,12 @@ function _start_ovn_services {
    if is_service_enabled ovs-vtep ; then
        _start_process "devstack@ovs-vtep.service"
    fi
-    if is_service_enabled q-ovn-metadata-agent; then
+    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent ; then
        _start_process "devstack@q-ovn-metadata-agent.service"
    fi
+    if is_service_enabled q-ovn-agent neutron-ovn-agent ; then
+        _start_process "devstack@q-ovn-agent.service"
+    fi
 }

 # start_ovn() - Start running processes, including screen
@@ -689,23 +744,26 @@ function start_ovn {
            local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_northd"
            local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_northd"

-            _run_process ovn-northd "$cmd" "$stop_cmd"
+            _run_process ovn-northd "$cmd" "$stop_cmd" "$STACK_GROUP" "root" "$OVN_RUNDIR"
        else
            _start_process "$OVN_NORTHD_SERVICE"
        fi

        # Wait for the service to be ready
-        wait_for_sock_file $OVS_RUNDIR/ovnnb_db.sock
-        wait_for_sock_file $OVS_RUNDIR/ovnsb_db.sock
+        # Check for socket and db files for both OVN NB and SB
+        wait_for_sock_file $OVN_RUNDIR/ovnnb_db.sock
+        wait_for_sock_file $OVN_RUNDIR/ovnsb_db.sock
+        wait_for_db_file $OVN_DATADIR/ovnnb_db.db
+        wait_for_db_file $OVN_DATADIR/ovnsb_db.db

        if is_service_enabled tls-proxy; then
-            sudo ovn-nbctl --db=unix:$OVS_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
-            sudo ovn-sbctl --db=unix:$OVS_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
+            sudo ovn-nbctl --db=unix:$OVN_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
+            sudo ovn-sbctl --db=unix:$OVN_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
        fi
-        sudo ovn-nbctl --db=unix:$OVS_RUNDIR/ovnnb_db.sock set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
-        sudo ovn-sbctl --db=unix:$OVS_RUNDIR/ovnsb_db.sock set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
-        sudo ovs-appctl -t $OVS_RUNDIR/ovnnb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
-        sudo ovs-appctl -t $OVS_RUNDIR/ovnsb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
+        sudo ovn-nbctl --db=unix:$OVN_RUNDIR/ovnnb_db.sock set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
+        sudo ovn-sbctl --db=unix:$OVN_RUNDIR/ovnsb_db.sock set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
+        sudo ovs-appctl -t $OVN_RUNDIR/ovnnb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
+        sudo ovs-appctl -t $OVN_RUNDIR/ovnsb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
    fi

    if is_service_enabled ovn-controller ; then
@@ -713,7 +771,7 @@ function start_ovn {
            local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_controller"
            local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_controller"

-            _run_process ovn-controller "$cmd" "$stop_cmd" "$STACK_GROUP" "root"
+            _run_process ovn-controller "$cmd" "$stop_cmd" "$STACK_GROUP" "root" "$OVN_RUNDIR"
        else
            _start_process "$OVN_CONTROLLER_SERVICE"
        fi
@@ -722,18 +780,24 @@ function start_ovn {
    if is_service_enabled ovn-controller-vtep ; then
        if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
            local cmd="$OVS_BINDIR/ovn-controller-vtep --log-file --pidfile --detach --ovnsb-db=$OVN_SB_REMOTE"
-            _run_process ovn-controller-vtep "$cmd" "" "$STACK_GROUP" "root"
+            _run_process ovn-controller-vtep "$cmd" "" "$STACK_GROUP" "root" "$OVN_RUNDIR"
        else
            _start_process "$OVN_CONTROLLER_VTEP_SERVICE"
        fi
    fi

-    if is_service_enabled q-ovn-metadata-agent; then
+    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
        run_process q-ovn-metadata-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_METADATA_BINARY --config-file $OVN_META_CONF"
        # Format logging
        setup_logging $OVN_META_CONF
    fi

+    if is_service_enabled q-ovn-agent neutron-ovn-agent; then
+        run_process q-ovn-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_AGENT_BINARY --config-file $OVN_AGENT_CONF"
+        # Format logging
+        setup_logging $OVN_AGENT_CONF
+    fi
+
    _start_ovn_services
 }

@@ -752,10 +816,18 @@ function _stop_process {
 }

 function stop_ovn {
-    if is_service_enabled q-ovn-metadata-agent; then
-        sudo pkill -9 -f haproxy || :
+    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
+        # pkill takes care not to kill itself, but it may kill its parent
+        # sudo unless we use the "ps | grep [f]oo" trick
+        sudo pkill -9 -f "[h]aproxy" || :
        _stop_process "devstack@q-ovn-metadata-agent.service"
    fi
+    if is_service_enabled q-ovn-agent neutron-ovn-agent; then
+        # pkill takes care not to kill itself, but it may kill its parent
+        # sudo unless we use the "ps | grep [f]oo" trick
+        sudo pkill -9 -f "[h]aproxy" || :
+        _stop_process "devstack@q-ovn-agent.service"
+    fi
    if is_service_enabled ovn-controller-vtep ; then
        _stop_process "$OVN_CONTROLLER_VTEP_SERVICE"
    fi
@@ -798,5 +870,5 @@ function cleanup_ovn {
        _cleanup $ovs_path
    fi

-    sudo rm -f $OVN_RUNDIR
+    sudo rm -rf $OVN_RUNDIR
 }
@@ -80,19 +80,6 @@ function _neutron_ovs_base_install_agent_packages {
        elif is_fedora; then
            restart_service openvswitch
            sudo systemctl enable openvswitch
-        elif is_suse; then
-            if [[ $DISTRO == "sle12" ]] && vercmp "$os_RELEASE" "<" "12.2" ; then
-                restart_service openvswitch-switch
-            else
-                # workaround for https://bugzilla.suse.com/show_bug.cgi?id=1085971
-                if [[ $DISTRO =~ "tumbleweed" ]]; then
-                    sudo sed -i -e "s,^OVS_USER_ID=.*,OVS_USER_ID='root:root'," /etc/sysconfig/openvswitch
-                fi
-                restart_service openvswitch || {
-                    journalctl -xe || :
-                    systemctl status openvswitch
-                }
-            fi
        fi
    fi
 }
@@ -33,9 +33,9 @@ function load_module {
    local fatal=$2

    if [ "$(trueorfalse True fatal)" == "True" ]; then
-        sudo modprobe $module || (dmesg && die $LINENO "FAILED TO LOAD $module")
+        sudo modprobe $module || (sudo dmesg && die $LINENO "FAILED TO LOAD $module")
    else
-        sudo modprobe $module || (echo "FAILED TO LOAD $module" && dmesg)
+        sudo modprobe $module || (echo "FAILED TO LOAD $module" && sudo dmesg)
    fi
 }

@@ -87,9 +87,15 @@ function prepare_for_ovs_compilation {

        install_package kernel-devel-$KERNEL_VERSION
        install_package kernel-headers-$KERNEL_VERSION
+        if is_service_enabled tls-proxy; then
+            install_package openssl-devel
+        fi

    elif is_ubuntu ; then
        install_package linux-headers-$KERNEL_VERSION
+        if is_service_enabled tls-proxy; then
+            install_package libssl-dev
+        fi
    fi
 }

@@ -97,7 +103,7 @@ function prepare_for_ovs_compilation {
 function load_ovs_kernel_modules {
    load_module openvswitch
    load_module vport-geneve False
-    dmesg | tail
+    sudo dmesg | tail
 }

 # reload_ovs_kernel_modules() - reload openvswitch kernel module
@@ -158,10 +164,8 @@ function compile_ovs {
    sudo make install
    if [[ "$build_modules" == "True" ]]; then
        sudo make INSTALL_MOD_DIR=kernel/net/openvswitch modules_install
-        reload_ovs_kernel_modules
-    else
-        load_ovs_kernel_modules
    fi
+    reload_ovs_kernel_modules

    cd $_pwd
 }
@@ -176,12 +180,6 @@ function action_openvswitch {
        ${action}_service openvswitch-switch
    elif is_fedora; then
        ${action}_service openvswitch
-    elif is_suse; then
-        if [[ $DISTRO == "sle12" ]] && [[ $os_RELEASE -lt 12.2 ]]; then
-            ${action}_service openvswitch-switch
-        else
-            ${action}_service openvswitch
-        fi
    fi
 }

@@ -47,7 +47,8 @@ Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True}
 # used for the network.  In case of ofagent, you should add the
 # corresponding entry to your OFAGENT_PHYSICAL_INTERFACE_MAPPINGS.
 # For openvswitch agent, you should add the corresponding entry to
-# your OVS_BRIDGE_MAPPINGS.
+# your OVS_BRIDGE_MAPPINGS and for OVN add the corresponding entry
+# to your OVN_BRIDGE_MAPPINGS.
 #
 # eg.  (ofagent)
 #    Q_USE_PROVIDERNET_FOR_PUBLIC=True
@@ -60,6 +61,11 @@ Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True}
 #    PUBLIC_PHYSICAL_NETWORK=public
 #    OVS_BRIDGE_MAPPINGS=public:br-ex
 #
+# eg.  (ovn agent)
+#    Q_USER_PROVIDERNET_FOR_PUBLIC=True
+#    PUBLIC_PHYSICAL_NETWORK=public
+#    OVN_BRIDGE_MAPPINGS=public:br-ex
+#
 # The provider-network-type defaults to flat, however, the values
 # PUBLIC_PROVIDERNET_TYPE and PUBLIC_PROVIDERNET_SEGMENTATION_ID could
 # be set to specify the parameters for an alternate network type.
@@ -147,10 +153,6 @@ function _neutron_get_ext_gw_interface {
 }

 function create_neutron_initial_network {
-    local project_id
-    project_id=$(openstack project list | grep " demo " | get_field 1)
-    die_if_not_set $LINENO project_id "Failure retrieving project_id for demo"
-
    # Allow drivers that need to create an initial network to do so here
    if type -p neutron_plugin_create_initial_network_profile > /dev/null; then
        neutron_plugin_create_initial_network_profile $PHYSICAL_NETWORK
@@ -170,15 +172,15 @@ function create_neutron_initial_network {
    if is_provider_network; then
        die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specify the PROVIDER_NETWORK_TYPE"
-        NET_ID=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" network create $PHYSICAL_NETWORK --provider-network-type $PROVIDER_NETWORK_TYPE --provider-physical-network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider-segment $SEGMENTATION_ID} --share | grep ' id ' | get_field 2)
-        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $project_id"
+        NET_ID=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" network create $PHYSICAL_NETWORK --provider-network-type $PROVIDER_NETWORK_TYPE --provider-physical-network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider-segment $SEGMENTATION_ID} --share -f value -c id)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            if [ -z $SUBNETPOOL_V4_ID ]; then
                fixed_range_v4=$FIXED_RANGE
            fi
-            SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $project_id"
+            SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} -f value -c id)
+            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME"
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
@@ -187,8 +189,8 @@ function create_neutron_initial_network {
            if [ -z $SUBNETPOOL_V6_ID ]; then
                fixed_range_v6=$IPV6_PROVIDER_FIXED_RANGE
            fi
-            IPV6_SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME $project_id"
+            IPV6_SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} -f value -c id)
+            die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME"
        fi

        if [[ $Q_AGENT == "openvswitch" ]]; then
@@ -197,17 +199,17 @@ function create_neutron_initial_network {
            sudo ip link set $PUBLIC_INTERFACE up
        fi
    else
-        NET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" network create "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
-        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME $project_id"
+        NET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" network create "$PRIVATE_NETWORK_NAME" -f value -c id)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Create IPv4 private subnet
-            SUBNET_ID=$(_neutron_create_private_subnet_v4 $project_id)
+            SUBNET_ID=$(_neutron_create_private_subnet_v4)
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Create IPv6 private subnet
-            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6 $project_id)
+            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6)
        fi
    fi

@@ -215,12 +217,12 @@ function create_neutron_initial_network {
        # Create a router, and add the private subnet as one of its interfaces
        if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
            # create a tenant-owned router.
-            ROUTER_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" router create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $project_id $Q_ROUTER_NAME"
+            ROUTER_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME"
        else
            # Plugin only supports creating a single router, which should be admin owned.
-            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $Q_ROUTER_NAME"
+            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME"
        fi

        EXTERNAL_NETWORK_FLAGS="--external"
@@ -229,9 +231,9 @@ function create_neutron_initial_network {
        fi
        # Create an external network, and a subnet. Configure the external network as router gw
        if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
-            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS --provider-network-type ${PUBLIC_PROVIDERNET_TYPE:-flat} ${PUBLIC_PROVIDERNET_SEGMENTATION_ID:+--provider-segment $PUBLIC_PROVIDERNET_SEGMENTATION_ID} --provider-physical-network ${PUBLIC_PHYSICAL_NETWORK} | grep ' id ' | get_field 2)
+            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS --provider-network-type ${PUBLIC_PROVIDERNET_TYPE:-flat} ${PUBLIC_PROVIDERNET_SEGMENTATION_ID:+--provider-segment $PUBLIC_PROVIDERNET_SEGMENTATION_ID} --provider-physical-network ${PUBLIC_PHYSICAL_NETWORK} -f value -c id)
        else
-            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS | grep ' id ' | get_field 2)
+            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS -f value -c id)
        fi
        die_if_not_set $LINENO EXT_NET_ID "Failure creating EXT_NET_ID for $PUBLIC_NETWORK_NAME"

@@ -249,7 +251,6 @@ function create_neutron_initial_network {

 # Create private IPv4 subnet
 function _neutron_create_private_subnet_v4 {
-    local project_id=$1
    if [ -z $SUBNETPOOL_V4_ID ]; then
        fixed_range_v4=$FIXED_RANGE
    fi
@@ -262,14 +263,13 @@ function _neutron_create_private_subnet_v4 {
    subnet_params+="${fixed_range_v4:+--subnet-range $fixed_range_v4} "
    subnet_params+="--network $NET_ID $PRIVATE_SUBNET_NAME"
    local subnet_id
-    subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params | grep ' id ' | get_field 2)
-    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet for $project_id"
+    subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params -f value -c id)
+    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet"
    echo $subnet_id
 }

 # Create private IPv6 subnet
 function _neutron_create_private_subnet_v6 {
-    local project_id=$1
    die_if_not_set $LINENO IPV6_RA_MODE "IPV6 RA Mode not set"
    die_if_not_set $LINENO IPV6_ADDRESS_MODE "IPV6 Address Mode not set"
    local ipv6_modes="--ipv6-ra-mode $IPV6_RA_MODE --ipv6-address-mode $IPV6_ADDRESS_MODE"
@@ -284,8 +284,8 @@ function _neutron_create_private_subnet_v6 {
    subnet_params+="${fixed_range_v6:+--subnet-range $fixed_range_v6} "
    subnet_params+="$ipv6_modes --network $NET_ID $IPV6_PRIVATE_SUBNET_NAME "
    local ipv6_subnet_id
-    ipv6_subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params | grep ' id ' | get_field 2)
-    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet for $project_id"
+    ipv6_subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params -f value -c id)
+    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet"
    echo $ipv6_subnet_id
 }

@@ -329,7 +329,7 @@ function _neutron_configure_router_v4 {
    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID

    # This logic is specific to using OVN or the l3-agent for layer 3
-    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-server) || is_service_enabled q-l3 neutron-l3; then
+    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-api) || is_service_enabled q-l3 neutron-l3; then
        # Configure and enable public bridge
        local ext_gw_interface="none"
        if is_neutron_ovs_base_plugin; then
@@ -378,7 +378,7 @@ function _neutron_configure_router_v6 {
    fi

    # This logic is specific to using OVN or the l3-agent for layer 3
-    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-server) || is_service_enabled q-l3 neutron-l3; then
+    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-api) || is_service_enabled q-l3 neutron-l3; then
        # if the Linux host considers itself to be a router then it will
        # ignore all router advertisements
        # Ensure IPv6 RAs are accepted on interfaces with a default route.
@@ -409,7 +409,10 @@ function _neutron_configure_router_v6 {
            ext_gw_interface=$(_neutron_get_ext_gw_interface)
            local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}

-            # Configure interface for public bridge
+            # Configure interface for public bridge by setting the interface
+            # to "up" in case the job is running entirely private network based
+            # testing.
+            sudo ip link set $ext_gw_interface up
            sudo ip -6 addr replace $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
            # Any IPv6 private subnet that uses the default IPV6 subnet pool
            # and that is plugged into the default router (Q_ROUTER_NAME) will
@@ -12,7 +12,7 @@ AGENT_METERING_BINARY="$NEUTRON_BIN_DIR/neutron-metering-agent"
 METERING_PLUGIN="neutron.services.metering.metering_plugin.MeteringPlugin"

 function neutron_agent_metering_configure_common {
-    _neutron_service_plugin_class_add $METERING_PLUGIN
+    neutron_service_plugin_class_add $METERING_PLUGIN
 }

 function neutron_agent_metering_configure_agent {
@@ -6,7 +6,7 @@ function configure_qos_service_plugin {


 function configure_qos_core_plugin {
-    configure_qos_$NEUTRON_CORE_PLUGIN
+    configure_qos_$Q_PLUGIN
 }


@@ -0,0 +1,10 @@
+#!/bin/bash
+
+function configure_segments_service_plugin {
+    neutron_service_plugin_class_add segments
+}
+
+function configure_segments_extension {
+    configure_segments_service_plugin
+}
+
@@ -53,11 +53,19 @@ NOVA_COND_CONF=$NOVA_CONF_DIR/nova.conf
 NOVA_CPU_CONF=$NOVA_CONF_DIR/nova-cpu.conf
 NOVA_FAKE_CONF=$NOVA_CONF_DIR/nova-fake.conf
 NOVA_API_DB=${NOVA_API_DB:-nova_api}
-NOVA_UWSGI=$NOVA_BIN_DIR/nova-api-wsgi
-NOVA_METADATA_UWSGI=$NOVA_BIN_DIR/nova-metadata-wsgi
+NOVA_UWSGI=nova.wsgi.osapi_compute:application
+NOVA_METADATA_UWSGI=nova.wsgi.metadata:application
 NOVA_UWSGI_CONF=$NOVA_CONF_DIR/nova-api-uwsgi.ini
 NOVA_METADATA_UWSGI_CONF=$NOVA_CONF_DIR/nova-metadata-uwsgi.ini

+# Allow forcing the stable compute uuid to something specific. This would be
+# done by deployment tools that pre-allocate the UUIDs, but it is also handy
+# for developers that need to re-stack a compute-only deployment multiple
+# times. Since the DB is non-local and not erased on an unstack, making it
+# stay the same each time is what developers want. Set to a uuid here or
+# leave it blank for default allocate-on-start behavior.
+NOVA_CPU_UUID=""
+
 # The total number of cells we expect. Must be greater than one and doesn't
 # count cell0.
 NOVA_NUM_CELLS=${NOVA_NUM_CELLS:-1}
@@ -75,6 +83,11 @@ NOVA_API_PASTE_INI=${NOVA_API_PASTE_INI:-$NOVA_CONF_DIR/api-paste.ini}
 # mean "use uwsgi" because we'll be always using uwsgi.
 NOVA_USE_MOD_WSGI=${NOVA_USE_MOD_WSGI:-True}

+# We do not need to report service status every 10s for devstack-like
+# deployments. In the gate this generates extra work for the services and the
+# database which are already taxed.
+NOVA_SERVICE_REPORT_INTERVAL=${NOVA_SERVICE_REPORT_INTERVAL:-120}
+
 if is_service_enabled tls-proxy; then
    NOVA_SERVICE_PROTOCOL="https"
 fi
@@ -97,30 +110,28 @@ NOVA_SERVICE_LISTEN_ADDRESS=${NOVA_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $SERVI
 METADATA_SERVICE_PORT=${METADATA_SERVICE_PORT:-8775}
 NOVA_ENABLE_CACHE=${NOVA_ENABLE_CACHE:-True}

+# Flag to set the oslo_policy.enforce_scope and oslo_policy.enforce_new_defaults.
+# This is used to disable the compute API policies scope and new defaults.
+# By Default, it is True.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+NOVA_ENFORCE_SCOPE=$(trueorfalse True NOVA_ENFORCE_SCOPE)
+
+if [[ $SERVICE_IP_VERSION == 6 ]]; then
+    NOVA_MY_IP="$HOST_IPV6"
+else
+    NOVA_MY_IP="$HOST_IP"
+fi
+
 # Option to enable/disable config drive
 # NOTE: Set ``FORCE_CONFIG_DRIVE="False"`` to turn OFF config drive
 FORCE_CONFIG_DRIVE=${FORCE_CONFIG_DRIVE:-"False"}

 # The following NOVA_FILTERS contains SameHostFilter and DifferentHostFilter with
 # the default filters.
-NOVA_FILTERS="AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter"
+NOVA_FILTERS="ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter"

 QEMU_CONF=/etc/libvirt/qemu.conf

-# Set default defaults here as some hypervisor drivers override these
-PUBLIC_INTERFACE_DEFAULT=br100
-# Set ``GUEST_INTERFACE_DEFAULT`` to some interface on the box so that
-# the default isn't completely crazy. This will match ``eth*``, ``em*``, or
-# the new ``p*`` interfaces, then basically picks the first
-# alphabetically. It's probably wrong, however it's less wrong than
-# always using ``eth0`` which doesn't exist on new Linux distros at all.
-GUEST_INTERFACE_DEFAULT=$(ip link \
-    | grep 'state UP' \
-    | awk '{print $2}' \
-    | sed 's/://' \
-    | grep ^[ep] \
-    | head -1)
-
 # ``NOVA_VNC_ENABLED`` can be used to forcibly enable VNC configuration.
 # In multi-node setups allows compute hosts to not run ``n-novnc``.
 NOVA_VNC_ENABLED=$(trueorfalse False NOVA_VNC_ENABLED)
@@ -162,6 +173,9 @@ NOVA_SHUTDOWN_TIMEOUT=${NOVA_SHUTDOWN_TIMEOUT:-0}
 # Whether to use Keystone unified limits instead of legacy quota limits.
 NOVA_USE_UNIFIED_LIMITS=$(trueorfalse False NOVA_USE_UNIFIED_LIMITS)

+# TB Cache Size in MiB for qemu guests
+NOVA_LIBVIRT_TB_CACHE_SIZE=${NOVA_LIBVIRT_TB_CACHE_SIZE:-0}
+
 # Functions
 # ---------

@@ -219,6 +233,9 @@ function cleanup_nova {
        done
        sudo iscsiadm --mode node --op delete || true

+        # Disconnect all nvmeof connections
+        sudo nvme disconnect-all || true
+
        # Clean out the instances directory.
        sudo rm -rf $NOVA_INSTANCES_PATH/*
    fi
@@ -234,8 +251,8 @@ function cleanup_nova {

    stop_process "n-api"
    stop_process "n-api-meta"
-    remove_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI"
-    remove_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI"
+    remove_uwsgi_config "$NOVA_UWSGI_CONF" "nova-api"
+    remove_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "nova-metadata"

    if [[ "$NOVA_BACKEND" == "LVM" ]]; then
        clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME
@@ -306,6 +323,7 @@ function configure_nova {
            fi
        fi

+        # Due to cinder bug #1966513 we ALWAYS need an initiator name for LVM
        # Ensure each compute host uses a unique iSCSI initiator
        echo InitiatorName=$(iscsi-iname) | sudo tee /etc/iscsi/initiatorname.iscsi

@@ -324,14 +342,30 @@ EOF

        # set chap algorithms.  The default chap_algorithm is md5 which will
        # not work under FIPS.
-        # FIXME(alee) For some reason, this breaks openeuler.  Openeuler devs should weigh in
-        # and determine the correct solution for openeuler here
-        if ! is_openeuler; then
-            iniset -sudo /etc/iscsi/iscsid.conf DEFAULT "node.session.auth.chap_algs" "SHA3-256,SHA256"
-        fi
+        iniset -sudo /etc/iscsi/iscsid.conf DEFAULT "node.session.auth.chap_algs" "SHA3-256,SHA256"

-        # ensure that iscsid is started, even when disabled by default
-        restart_service iscsid
+        if [[ $CINDER_TARGET_HELPER != 'nvmet' ]]; then
+            # ensure that iscsid is started, even when disabled by default
+            restart_service iscsid
+
+        # For NVMe-oF we need different packages that many not be present
+        else
+            install_package nvme-cli
+            sudo modprobe nvme-fabrics
+
+            # Ensure NVMe is ready and create the Soft-RoCE device over the networking interface
+            if [[ $CINDER_TARGET_PROTOCOL == 'nvmet_rdma' ]]; then
+                sudo modprobe nvme-rdma
+                iface=${HOST_IP_IFACE:-`ip -br -$SERVICE_IP_VERSION a | grep $NOVA_MY_IP | awk '{print $1}'`}
+                if ! sudo rdma link | grep $iface ; then
+                    sudo rdma link add rxe_$iface type rxe netdev $iface
+                fi
+            elif [[ $CINDER_TARGET_PROTOCOL == 'nvmet_tcp' ]]; then
+                sudo modprobe nvme-tcp
+            else  # 'nvmet_fc'
+                sudo modprobe nvme-fc
+            fi
+        fi
    fi

    # Rebuild the config file from scratch
@@ -422,11 +456,7 @@ function create_nova_conf {
    iniset $NOVA_CONF filter_scheduler enabled_filters "$NOVA_FILTERS"
    iniset $NOVA_CONF scheduler workers "$API_WORKERS"
    iniset $NOVA_CONF neutron default_floating_pool "$PUBLIC_NETWORK_NAME"
-    if [[ $SERVICE_IP_VERSION == 6 ]]; then
-        iniset $NOVA_CONF DEFAULT my_ip "$HOST_IPV6"
-    else
-        iniset $NOVA_CONF DEFAULT my_ip "$HOST_IP"
-    fi
+    iniset $NOVA_CONF DEFAULT my_ip "$NOVA_MY_IP"
    iniset $NOVA_CONF DEFAULT instance_name_template "${INSTANCE_NAME_PREFIX}%08x"
    iniset $NOVA_CONF DEFAULT osapi_compute_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
    iniset $NOVA_CONF DEFAULT metadata_listen "$NOVA_SERVICE_LISTEN_ADDRESS"
@@ -434,8 +464,11 @@ function create_nova_conf {

    iniset $NOVA_CONF key_manager backend nova.keymgr.conf_key_mgr.ConfKeyManager

-    if is_fedora || is_suse; then
-        # nova defaults to /usr/local/bin, but fedora and suse pip like to
+    iniset $NOVA_CONF DEFAULT report_interval $NOVA_SERVICE_REPORT_INTERVAL
+    iniset $NOVA_CONF DEFAULT service_down_time $(($NOVA_SERVICE_REPORT_INTERVAL * 6))
+
+    if is_fedora; then
+        # nova defaults to /usr/local/bin, but fedora pip like to
        # install things in /usr/bin
        iniset $NOVA_CONF DEFAULT bindir "/usr/bin"
    fi
@@ -473,6 +506,13 @@ function create_nova_conf {
            NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/,metadata//")
        fi
        iniset $NOVA_CONF DEFAULT enabled_apis "$NOVA_ENABLED_APIS"
+        if [[ "$NOVA_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]]; then
+            iniset $NOVA_CONF oslo_policy enforce_new_defaults True
+            iniset $NOVA_CONF oslo_policy enforce_scope True
+        else
+            iniset $NOVA_CONF oslo_policy enforce_new_defaults False
+            iniset $NOVA_CONF oslo_policy enforce_scope False
+        fi
        if is_service_enabled tls-proxy && [ "$NOVA_USE_MOD_WSGI" == "False" ]; then
            # Set the service port for a proxy to take the original
            iniset $NOVA_CONF DEFAULT osapi_compute_listen_port "$NOVA_SERVICE_PORT_INT"
@@ -502,7 +542,7 @@ function create_nova_conf {

    # nova defaults to genisoimage but only mkisofs is available for 15.0+
    # rhel provides mkisofs symlink to genisoimage or xorriso appropiately
-    if is_suse || is_fedora; then
+    if is_fedora; then
        iniset $NOVA_CONF DEFAULT mkisofs_cmd /usr/bin/mkisofs
    fi

@@ -512,11 +552,11 @@ function create_nova_conf {
    iniset $NOVA_CONF upgrade_levels compute "auto"

    if is_service_enabled n-api; then
-        write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute"
+        write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute" "" "nova-api"
    fi

    if is_service_enabled n-api-meta; then
-        write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}"
+        write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}" "nova-metadata"
    fi

    if is_service_enabled ceilometer; then
@@ -600,7 +640,7 @@ function configure_placement_nova_compute {
    local conf=${1:-$NOVA_CONF}
    iniset $conf placement auth_type "password"
    iniset $conf placement auth_url "$KEYSTONE_SERVICE_URI"
-    iniset $conf placement username placement
+    iniset $conf placement username nova
    iniset $conf placement password "$SERVICE_PASSWORD"
    iniset $conf placement user_domain_name "$SERVICE_DOMAIN_NAME"
    iniset $conf placement project_name "$SERVICE_TENANT_NAME"
@@ -903,8 +943,23 @@ function install_nova {
        # a websockets/html5 or flash powered VNC console for vm instances
        NOVNC_FROM_PACKAGE=$(trueorfalse False NOVNC_FROM_PACKAGE)
        if [ "$NOVNC_FROM_PACKAGE" = "True" ]; then
+            # Installing novnc on Debian bullseye breaks the global pip
+            # install. This happens because novnc pulls in distro cryptography
+            # which will be prefered by distro pip, but if anything has
+            # installed pyOpenSSL from pypi (keystone) that is not compatible
+            # with distro cryptography. Fix this by installing
+            # python3-openssl (pyOpenSSL) from the distro which pip will prefer
+            # on Debian. Ubuntu has inverse problems so we only do this for
+            # Debian.
+            local novnc_packages
+            novnc_packages="novnc"
+            GetOSVersion
+            if [[ "$os_VENDOR" = "Debian" ]] ; then
+                novnc_packages="$novnc_packages python3-openssl"
+            fi
+
            NOVNC_WEB_DIR=/usr/share/novnc
-            install_package novnc
+            install_package $novnc_packages
        else
            NOVNC_WEB_DIR=$DEST/novnc
            git_clone $NOVNC_REPO $NOVNC_WEB_DIR $NOVNC_BRANCH
@@ -1000,6 +1055,10 @@ function start_nova_compute {
    # by the compute process.
    configure_console_compute

+    # Set rebuild timeout longer for BFV instances because we likely have
+    # slower disk than expected. Default is 20s/GB
+    iniset $NOVA_CPU_CONF DEFAULT reimage_timeout_per_gb 180
+
    # Configure the OVSDB connection for os-vif
    if [ -n "$OVSDB_SERVER_LOCAL_HOST" ]; then
        iniset $NOVA_CPU_CONF os_vif_ovs ovsdb_connection "tcp:$OVSDB_SERVER_LOCAL_HOST:6640"
@@ -1010,7 +1069,14 @@ function start_nova_compute {
        iniset $NOVA_CPU_CONF workarounds libvirt_disable_apic True
    fi

+    if [[ "$NOVA_CPU_UUID" ]]; then
+        echo -n $NOVA_CPU_UUID > $NOVA_CONF_DIR/compute_id
+    fi
+
    if [[ "$VIRT_DRIVER" = 'libvirt' ]]; then
+        if [ ${NOVA_LIBVIRT_TB_CACHE_SIZE} -gt 0 ]; then
+            iniset $NOVA_CPU_CONF libvirt tb_cache_size ${NOVA_LIBVIRT_TB_CACHE_SIZE}
+        fi
        # The group **$LIBVIRT_GROUP** is added to the current user in this script.
        # ``sg`` is used in run_process to execute nova-compute as a member of the
        # **$LIBVIRT_GROUP** group.
@@ -1045,7 +1111,6 @@ function start_nova_rest {
    local old_path=$PATH
    export PATH=$NOVA_BIN_DIR:$PATH

-    local api_cell_conf=$NOVA_CONF
    local compute_cell_conf=$NOVA_CONF

    run_process n-sch "$NOVA_BIN_DIR/nova-scheduler --config-file $compute_cell_conf"
@@ -69,12 +69,12 @@ function install_libvirt {
            $REQUIREMENTS_DIR/upper-constraints.txt -- libvirt-python

    if is_ubuntu; then
-        install_package qemu-system libvirt-clients libvirt-daemon-system libvirt-dev python3-libvirt
+        install_package qemu-system libvirt-clients libvirt-daemon-system libvirt-dev python3-libvirt systemd-coredump
        if is_arch "aarch64"; then
            install_package qemu-efi
        fi
        #pip_install_gr <there-si-no-guestfs-in-pypi>
-    elif is_fedora || is_suse; then
+    elif is_fedora; then

        # Optionally enable the virt-preview repo when on Fedora
        if [[ $DISTRO =~ f[0-9][0-9] ]] && [[ ${ENABLE_FEDORA_VIRT_PREVIEW_REPO} == "True" ]]; then
@@ -82,11 +82,17 @@ function install_libvirt {
            sudo dnf copr enable -y @virtmaint-sig/virt-preview
        fi

+        if is_openeuler; then
+            qemu_package=qemu
+        else
+            qemu_package=qemu-kvm
+        fi
+
        # Note that in CentOS/RHEL this needs to come from the RDO
        # repositories (qemu-kvm-ev ... which provides this package)
        # as the base system version is too old.  We should have
        # pre-installed these
-        install_package qemu-kvm
+        install_package $qemu_package
        install_package libvirt libvirt-devel python3-libvirt

        if is_arch "aarch64"; then
@@ -115,8 +121,8 @@ cgroup_device_acl = [
 EOF
    fi

-    if is_fedora || is_suse; then
-        # Starting with fedora 18 and opensuse-12.3 enable stack-user to
+    if is_fedora; then
+        # Starting with fedora 18 enable stack-user to
        # virsh -c qemu:///system by creating a policy-kit rule for
        # stack-user using the new Javascript syntax
        rules_dir=/etc/polkit-1/rules.d
@@ -53,6 +53,10 @@ function configure_nova_hypervisor {
        iniset $NOVA_CONF ironic project_domain_id default
        iniset $NOVA_CONF ironic project_name demo
    fi
+    if is_ironic_sharded; then
+        iniset $NOVA_CONF ironic shard $IRONIC_SHARD_1_NAME
+    fi
+
    iniset $NOVA_CONF ironic user_domain_id default
    iniset $NOVA_CONF ironic region_name $REGION_NAME

@@ -56,6 +56,10 @@ function configure_nova_hypervisor {
    # arm64-specific configuration
    if is_arch "aarch64"; then
        iniset $NOVA_CONF libvirt cpu_mode "host-passthrough"
+        # NOTE(mnaser): We cannot have `cpu_model` set if the `cpu_mode` is
+        #               set to `host-passthrough`, or `nova-compute` refuses to
+        #               start.
+        inidelete $NOVA_CONF libvirt cpu_model
    fi

    if isset ENABLE_FILE_INJECTION; then
@@ -114,9 +118,6 @@ function install_nova_hypervisor {
                    sudo dpkg-statoverride --add --update $STAT_OVERRIDE
                fi
            done
-        elif is_suse; then
-            # Workaround for missing dependencies in python-libguestfs
-            install_package python-libguestfs guestfs-data augeas augeas-lenses
        elif is_fedora; then
            install_package python3-libguestfs
        fi
@@ -1,10 +1,5 @@
 #!/bin/bash

-# support vsctl or native.
-# until bug #1929446 is resolved we override the os-vif default
-# and fall back to the legacy "vsctl" driver.
-OS_VIF_OVS_OVSDB_INTERFACE=${OS_VIF_OVS_OVSDB_INTERFACE:="vsctl"}
-
 function is_ml2_ovs {
    if [[ "${Q_AGENT}" == "openvswitch" ]]; then
        echo "True"
@@ -19,11 +14,9 @@ OS_VIF_OVS_ISOLATE_VIF=$(trueorfalse False OS_VIF_OVS_ISOLATE_VIF)

 function configure_os_vif {
    if [[ -e ${NOVA_CONF} ]]; then
-        iniset ${NOVA_CONF} os_vif_ovs ovsdb_interface ${OS_VIF_OVS_OVSDB_INTERFACE}
        iniset ${NOVA_CONF} os_vif_ovs isolate_vif ${OS_VIF_OVS_ISOLATE_VIF}
    fi
    if [[ -e ${NEUTRON_CONF} ]]; then
-        iniset ${NEUTRON_CONF} os_vif_ovs ovsdb_interface ${OS_VIF_OVS_OVSDB_INTERFACE}
        iniset ${NEUTRON_CONF} os_vif_ovs isolate_vif ${OS_VIF_OVS_ISOLATE_VIF}
    fi
 }
@@ -48,6 +48,12 @@ fi
 PLACEMENT_SERVICE_PROTOCOL=${PLACEMENT_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 PLACEMENT_SERVICE_HOST=${PLACEMENT_SERVICE_HOST:-$SERVICE_HOST}

+# Flag to set the oslo_policy.enforce_scope and oslo_policy.enforce_new_defaults.
+# This is used to switch the Placement API policies scope and new defaults.
+# By Default, these flag are False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+PLACEMENT_ENFORCE_SCOPE=$(trueorfalse False PLACEMENT_ENFORCE_SCOPE)
+
 # Functions
 # ---------

@@ -62,7 +68,7 @@ function is_placement_enabled {
 # runs that a clean run would need to clean up
 function cleanup_placement {
    sudo rm -f $(apache_site_config_for placement-api)
-    remove_uwsgi_config "$PLACEMENT_UWSGI_CONF" "$PLACEMENT_UWSGI"
+    remove_uwsgi_config "$PLACEMENT_UWSGI_CONF" "placement-api"
 }

 # _config_placement_apache_wsgi() - Set WSGI config files
@@ -111,6 +117,13 @@ function configure_placement {
    else
        _config_placement_apache_wsgi
    fi
+    if [[ "$PLACEMENT_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]]; then
+        iniset $PLACEMENT_CONF oslo_policy enforce_new_defaults True
+        iniset $PLACEMENT_CONF oslo_policy enforce_scope True
+    else
+        iniset $PLACEMENT_CONF oslo_policy enforce_new_defaults False
+        iniset $PLACEMENT_CONF oslo_policy enforce_scope False
+    fi
 }

 # create_placement_accounts() - Set up required placement accounts
@@ -52,20 +52,7 @@ function install_rpc_backend {
    if is_service_enabled rabbit; then
        # Install rabbitmq-server
        install_package rabbitmq-server
-        if is_suse; then
-            install_package rabbitmq-server-plugins
-            # the default systemd socket activation only listens on the loopback interface
-            # which causes rabbitmq to try to start its own epmd
-            sudo mkdir -p /etc/systemd/system/epmd.socket.d
-            cat <<EOF | sudo tee /etc/systemd/system/epmd.socket.d/ports.conf >/dev/null
-[Socket]
-ListenStream=
-ListenStream=[::]:4369
-EOF
-            sudo systemctl daemon-reload
-            sudo systemctl restart epmd.socket epmd.service
-        fi
-        if is_fedora || is_suse; then
+        if is_fedora; then
            # NOTE(jangutter): If rabbitmq is not running (as in a fresh
            # install) then rabbit_setuser triggers epmd@0.0.0.0.socket with
            # socket activation. This fails the first time and does not get
--- a/Show More
+++ b/Show More