Merge "Replace deprecated datetime.utcnow()"

Merge "Globally skip devstack job for pre-commit config update"
Merge "Add image format enforcement toggle"
2024-10-25 08:38:40 +00:00 · 2024-10-25 07:56:22 +00:00 · 2024-10-25 07:56:19 +00:00 · 2024-10-24 00:54:28 +09:00 · 2024-10-21 14:22:36 +00:00 · 2024-10-19 12:55:43 +00:00
152 changed files with 4586 additions and 3251 deletions
@@ -38,3 +38,5 @@ stack-screenrc
 userrc_early
 AUTHORS
 ChangeLog
+tools/dbcounter/build/
+tools/dbcounter/dbcounter.egg-info/
@@ -1,16 +1,18 @@
- pragma:
-   # NOTE(gtema): this is required for the changes in SDK feature/r1 branch to
-   # be using devstack
-   # TODO(gtema): delete this once r1 branch is merged into master
-    implied-branches:
-      - master
-      - feature/r1
-
 - nodeset:
-    name: openstack-single-node
+    name: openstack-single-node-jammy
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: ubuntu-jammy
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: openstack-single-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
    groups:
      - name: tempest
        nodes:
@@ -37,30 +39,10 @@
          - controller

 - nodeset:
-    name: openstack-single-node-xenial
+    name: devstack-single-node-centos-9-stream
    nodes:
      - name: controller
-        label: ubuntu-xenial
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-7
-    nodes:
-      - name: controller
-        label: centos-7
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-8-stream
-    nodes:
-      - name: controller
-        label: centos-8-stream
+        label: centos-9-stream
    groups:
      - name: tempest
        nodes:
@@ -77,22 +59,105 @@
          - controller

 - nodeset:
-    name: devstack-single-node-fedora-latest
+    name: devstack-single-node-debian-bookworm
    nodes:
      - name: controller
-        label: fedora-34
+        label: debian-bookworm
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+# Note(sean-k-mooney): this is still used by horizon for
+# horizon-integration-tests, horizon-integration-pytest and
+# horizon-ui-pytest, remove when horizon is updated.
+- nodeset:
+    name: devstack-single-node-debian-bullseye
+    nodes:
+      - name: controller
+        label: debian-bullseye
    groups:
      - name: tempest
        nodes:
          - controller

 - nodeset:
-    name: openstack-two-node
+    name: devstack-single-node-rockylinux-9
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: rockylinux-9
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: openstack-two-node-centos-9-stream
+    nodes:
+      - name: controller
+        label: centos-9-stream
      - name: compute1
-        label: ubuntu-xenial
+        label: centos-9-stream
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
+- nodeset:
+    name: openstack-two-node-jammy
+    nodes:
+      - name: controller
+        label: ubuntu-jammy
+      - name: compute1
+        label: ubuntu-jammy
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
+- nodeset:
+    name: openstack-two-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
+      - name: compute1
+        label: ubuntu-noble
    groups:
      # Node where tests are executed and test results collected
      - name: tempest
@@ -176,36 +241,6 @@
        nodes:
          - compute1

- nodeset:
-    name: openstack-two-node-xenial
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-      - name: compute1
-        label: ubuntu-xenial
-    groups:
-      # Node where tests are executed and test results collected
-      - name: tempest
-        nodes:
-          - controller
-      # Nodes running the compute service
-      - name: compute
-        nodes:
-          - controller
-          - compute1
-      # Nodes that are not the controller
-      - name: subnode
-        nodes:
-          - compute1
-      # Switch node for multinode networking setup
-      - name: switch
-        nodes:
-          - controller
-      # Peer nodes for multinode networking setup
-      - name: peers
-        nodes:
-          - compute1
-
 - nodeset:
    name: openstack-three-node-focal
    nodes:
@@ -278,7 +313,7 @@

 - job:
    name: devstack-base
-    parent: multinode
+    parent: openstack-multinode-fips
    abstract: true
    description: |
      Base abstract Devstack job.
@@ -293,7 +328,6 @@
    required-projects:
      - opendev.org/openstack/devstack
    roles:
-      - zuul: opendev.org/openstack/devstack-gate
      - zuul: opendev.org/openstack/openstack-zuul-jobs
    vars:
      devstack_localrc:
@@ -327,8 +361,10 @@
        '{{ devstack_log_dir }}/devstacklog.txt.summary': logs
        '{{ devstack_log_dir }}/tcpdump.pcap': logs
        '{{ devstack_log_dir }}/worlddump-latest.txt': logs
+        '{{ devstack_log_dir }}/qemu.coredump': logs
        '{{ devstack_full_log}}': logs
        '{{ stage_dir }}/verify_tempest_conf.log': logs
+        '{{ stage_dir }}/performance.json': logs
        '{{ stage_dir }}/apache': logs
        '{{ stage_dir }}/apache_config': logs
        '{{ stage_dir }}/etc': logs
@@ -337,6 +373,7 @@
        /var/log/mysql: logs
        /var/log/libvirt: logs
        /etc/libvirt: logs
+        /etc/lvm: logs
        /etc/sudoers: logs
        /etc/sudoers.d: logs
        '{{ stage_dir }}/iptables.txt': logs
@@ -347,6 +384,7 @@
        '{{ stage_dir }}/rpm-qa.txt': logs
        '{{ stage_dir }}/core': logs
        '{{ stage_dir }}/listen53.txt': logs
+        '{{ stage_dir }}/services.txt': logs
        '{{ stage_dir }}/deprecations.log': logs
        '{{ stage_dir }}/audit.log': logs
        /etc/ceph: logs
@@ -394,6 +432,8 @@
      - ^releasenotes/.*$
      # Translations
      - ^.*/locale/.*po$
+      # pre-commit config
+      - ^.pre-commit-config.yaml$

 - job:
    name: devstack-minimal
@@ -401,7 +441,7 @@
    description: |
      Minimal devstack base job, intended for use by jobs that need
      less than the normal minimum set of required-projects.
-    nodeset: openstack-single-node-focal
+    nodeset: openstack-single-node-jammy
    required-projects:
      - opendev.org/openstack/requirements
    vars:
@@ -412,17 +452,21 @@
        PUBLIC_BRIDGE_MTU: '{{ external_bridge_mtu }}'
      devstack_services:
        # Shared services
-        dstat: true
+        dstat: false
        etcd3: true
        memory_tracker: true
+        file_tracker: true
        mysql: true
        rabbit: true
+        openstack-cli-server: true
    group-vars:
      subnode:
        devstack_services:
          # Shared services
-          dstat: true
+          dstat: false
          memory_tracker: true
+          file_tracker: true
+          openstack-cli-server: true
        devstack_localrc:
          # Multinode specific settings
          HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}"
@@ -468,8 +512,17 @@
      - opendev.org/openstack/nova
      - opendev.org/openstack/placement
      - opendev.org/openstack/swift
+      - opendev.org/openstack/os-test-images
    timeout: 7200
    vars:
+      # based on observation of the integrated gate
+      # tempest-integrated-compute was only using ~1.7GB of swap
+      # when zswap and the host turning are enabled that increase
+      # slightly to ~2GB. we are setting the swap size to 8GB to
+      # be safe and account for more complex scenarios.
+      # we should revisit this value after some time to see if we
+      # can reduce it.
+      configure_swap_size: 8192
      devstack_localrc:
        # Common OpenStack services settings
        SWIFT_REPLICAS: 1
@@ -478,6 +531,26 @@
        DEBUG_LIBVIRT_COREDUMPS: true
        NOVA_VNC_ENABLED: true
        OVN_DBS_LOG_LEVEL: dbg
+        # tune the host to optimize memory usage and hide io latency
+        # these setting will configure the kernel to treat the host page
+        # cache and swap with equal priority, and prefer deferring writes
+        # changing the default swappiness, dirty_ratio and
+        # the vfs_cache_pressure
+        ENABLE_SYSCTL_MEM_TUNING: true
+        # the net tuning optimizes ipv4 tcp fast open and config the default
+        # qdisk policy to pfifo_fast which effectively disable all qos.
+        # this minimizes the cpu load of the host network stack
+        ENABLE_SYSCTL_NET_TUNING: true
+        # zswap allows the kernel to compress pages in memory before swapping
+        # them to disk. this can reduce the amount of swap used and improve
+        # performance. effectively this trades a small amount of cpu for an
+        # increase in swap performance by reducing the amount of data
+        # written to disk. the overall speedup is proportional to the
+        # compression ratio and the speed of the swap device.
+        # NOTE: this option is ignored when not using nova with the libvirt
+        # virt driver.
+        NOVA_LIBVIRT_TB_CACHE_SIZE: 128
+        ENABLE_ZSWAP: true
      devstack_local_conf:
        post-config:
          $NEUTRON_CONF:
@@ -487,9 +560,10 @@
        # Core services enabled for this branch.
        # This list replaces the test-matrix.
        # Shared services
-        dstat: true
+        dstat: false
        etcd3: true
        memory_tracker: true
+        file_tracker: true
        mysql: true
        rabbit: true
        tls-proxy: true
@@ -524,7 +598,6 @@
        c-bak: true
        c-sch: true
        c-vol: true
-        cinder: true
        # Services we don't need.
        # This section is not really needed, it's for readability.
        horizon: false
@@ -538,8 +611,9 @@
          # Core services enabled for this branch.
          # This list replaces the test-matrix.
          # Shared services
-          dstat: true
+          dstat: false
          memory_tracker: true
+          file_tracker: true
          tls-proxy: true
          # Nova services
          n-cpu: true
@@ -569,16 +643,38 @@
          Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
          NOVA_VNC_ENABLED: true
          ENABLE_CHASSIS_AS_GW: false
+          # tune the host to optimize memory usage and hide io latency
+          # these setting will configure the kernel to treat the host page
+          # cache and swap with equal priority, and prefer deferring writes
+          # changing the default swappiness, dirty_ratio and
+          # the vfs_cache_pressure
+          ENABLE_SYSCTL_MEM_TUNING: true
+          # the net tuning optimizes ipv4 tcp fast open and config the default
+          # qdisk policy to pfifo_fast which effectively disable all qos.
+          # this minimizes the cpu load of the host network stack
+          ENABLE_SYSCTL_NET_TUNING: true
+          # zswap allows the kernel to compress pages in memory before swapping
+          # them to disk. this can reduce the amount of swap used and improve
+          # performance. effectivly this trades a small amount of cpu for an
+          # increase in swap performance by reducing the amount of data
+          # written to disk. the overall speedup is porportional to the
+          # compression ratio and the speed of the swap device.
+          ENABLE_ZSWAP: true
+          # NOTE: this option is ignored when not using nova with the libvirt
+          # virt driver.
+          NOVA_LIBVIRT_TB_CACHE_SIZE: 128

 - job:
    name: devstack-ipv6
    parent: devstack
    description: |
-      Devstack single node job for integration gate with IPv6.
+      Devstack single node job for integration gate with IPv6,
+      all services and tunnels using IPv6 addresses.
    vars:
      devstack_localrc:
        SERVICE_IP_VERSION: 6
        SERVICE_HOST: ""
+        TUNNEL_IP_VERSION: 6

 - job:
    name: devstack-enforce-scope
@@ -587,14 +683,12 @@
      This job runs the devstack with scope checks enabled.
    vars:
      devstack_localrc:
-        # Keep enabeling the services here to run with system scope
-        CINDER_ENFORCE_SCOPE: true
-        GLANCE_ENFORCE_SCOPE: true
+        ENFORCE_SCOPE: true

 - job:
    name: devstack-multinode
    parent: devstack
-    nodeset: openstack-two-node-focal
+    nodeset: openstack-two-node-jammy
    description: |
      Simple multinode test to verify multinode functionality on devstack side.
      This is not meant to be used as a parent job.
@@ -604,25 +698,95 @@
 # and these platforms don't have the round-the-clock support to avoid
 # becoming blockers in that situation.
 - job:
-    name: devstack-platform-centos-8-stream
+    name: devstack-platform-centos-9-stream
    parent: tempest-full-py3
-    description: CentOS 8 Stream platform test
-    nodeset: devstack-single-node-centos-8-stream
+    description: CentOS 9 Stream platform test
+    nodeset: devstack-single-node-centos-9-stream
+    timeout: 9000
    voting: false
+
+- job:
+    name: devstack-platform-debian-bookworm
+    parent: tempest-full-py3
+    description: Debian Bookworm platform test
+    nodeset: devstack-single-node-debian-bookworm
    timeout: 9000
    vars:
      configure_swap_size: 4096

 - job:
-    name: devstack-async
+    name: devstack-platform-rocky-blue-onyx
    parent: tempest-full-py3
-    description: Async mode enabled
+    description: Rocky Linux 9 Blue Onyx platform test
+    nodeset: devstack-single-node-rockylinux-9
+    timeout: 9000
+    # NOTE(danms): This has been failing lately with some repository metadata
+    # errors. We're marking this as non-voting until it appears to have
+    # stabilized:
+    # https://zuul.openstack.org/builds?job_name=devstack-platform-rocky-blue-onyx&skip=0
+    voting: false
+    vars:
+      configure_swap_size: 4096
+
+- job:
+    name: devstack-platform-ubuntu-noble
+    parent: tempest-full-py3
+    description: Ubuntu 24.04 LTS (noble) platform test
+    nodeset: openstack-single-node-noble
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovn-source
+    parent: devstack-platform-ubuntu-jammy
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVN from source)
    voting: false
    vars:
      devstack_localrc:
-        DEVSTACK_PARALLEL: True
-      zuul_copy_output:
-        /opt/stack/async: logs
+        OVN_BUILD_FROM_SOURCE: True
+        OVN_BRANCH: "v21.06.0"
+        OVS_BRANCH: "a4b04276ab5934d087669ff2d191a23931335c87"
+        OVS_SYSCONFDIR: "/usr/local/etc/openvswitch"
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovs
+    parent: tempest-full-py3
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVS)
+    nodeset: openstack-single-node-jammy
+    voting: false
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+      devstack_localrc:
+        Q_AGENT: openvswitch
+        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
+        Q_ML2_TENANT_NETWORK_TYPE: vxlan
+      devstack_services:
+        # Disable OVN services
+        ovn-northd: false
+        ovn-controller: false
+        ovs-vswitchd: false
+        ovsdb-server: false
+        # Disable Neutron ML2/OVN services
+        q-ovn-metadata-agent: false
+        # Enable Neutron ML2/OVS services
+        q-agt: true
+        q-dhcp: true
+        q-l3: true
+        q-meta: true
+        q-metering: true
+    group-vars:
+      subnode:
+        devstack_services:
+          # Disable OVN services
+          ovn-controller: false
+          ovs-vswitchd: false
+          ovsdb-server: false
+          # Disable Neutron ML2/OVN services
+          q-ovn-metadata-agent: false
+          # Enable Neutron ML2/OVS services
+          q-agt: true

 - job:
    name: devstack-no-tls-proxy
@@ -635,23 +799,6 @@
      devstack_services:
        tls-proxy: false

- job:
-    name: devstack-platform-fedora-latest
-    parent: tempest-full-py3
-    description: Fedora latest platform test
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-
- job:
-    name: devstack-platform-fedora-latest-virt-preview
-    parent: tempest-full-py3
-    description: Fedora latest platform test using the virt-preview repo.
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
-    vars:
-      devstack_localrc:
-        ENABLE_FEDORA_VIRT_PREVIEW_REPO: true
-
 - job:
    name: devstack-tox-base
    parent: devstack
@@ -708,7 +855,7 @@

 - job:
    name: devstack-unit-tests
-    nodeset: ubuntu-focal
+    nodeset: ubuntu-jammy
    description: |
      Runs unit tests on devstack project.

@@ -725,9 +872,12 @@
        - devstack
        - devstack-ipv6
        - devstack-enforce-scope
-        - devstack-platform-fedora-latest
-        - devstack-platform-centos-8-stream
-        - devstack-async
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
        - devstack-multinode
        - devstack-unit-tests
        - openstack-tox-bashate
@@ -741,10 +891,6 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-linuxbridge-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - neutron-ovn-tempest-ovs-release:
            voting: false
            irrelevant-files:
@@ -771,6 +917,11 @@
      jobs:
        - devstack
        - devstack-ipv6
+        - devstack-platform-debian-bookworm
+        - devstack-platform-ubuntu-noble
+        # NOTE(danms): Disabled due to instability, see comment in the job
+        # definition above.
+        # - devstack-platform-rocky-blue-onyx
        - devstack-enforce-scope
        - devstack-multinode
        - devstack-unit-tests
@@ -779,10 +930,6 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-linuxbridge-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - ironic-tempest-bios-ipmi-direct-tinyipa
        - swift-dsvm-functional
        - grenade:
@@ -806,7 +953,9 @@
    # pruned.
    #
    # * nova-next: maintained by nova for unreleased/undefaulted
-    #    things
+    #    things, this job is not experimental but often is used to test
+    #    things that are not yet production ready or to test what will be
+    #    the new default after a deprecation period has ended.
    # * neutron-fullstack-with-uwsgi: maintained by neutron for fullstack test
    #    when neutron-api is served by uwsgi, it's in exprimental for testing.
    #    the next cycle we can remove this  job if things turn out to be
@@ -814,14 +963,10 @@
    # * neutron-functional-with-uwsgi: maintained by neutron for functional
    #    test. Next cycle we can remove this one if things turn out to be
    #    stable engouh with uwsgi.
-    # * neutron-tempest-with-uwsgi: maintained by neutron for tempest test.
+    # * neutron-ovn-tempest-with-uwsgi: maintained by neutron for tempest test.
    #    Next cycle we can remove this if everything run out stable enough.
-    # * nova-multi-cell: maintained by nova and currently non-voting in the
+    # * nova-multi-cell: maintained by nova and now is voting in the
    #    check queue for nova changes but relies on devstack configuration
-    # * devstack-platform-fedora-latest-virt-preview: Maintained by lyarwood
-    #    for Nova to allow early testing of the latest versions of Libvirt and
-    #    QEMU. Should only graduate out of experimental if it ever moves into
-    #    the check queue for Nova.

    experimental:
      jobs:
@@ -829,7 +974,7 @@
        - nova-next
        - neutron-fullstack-with-uwsgi
        - neutron-functional-with-uwsgi
-        - neutron-tempest-with-uwsgi
+        - neutron-ovn-tempest-with-uwsgi
        - devstack-plugin-ceph-tempest-py3:
            irrelevant-files:
              - ^.*\.rst$
@@ -850,8 +995,15 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - devstack-platform-fedora-latest-virt-preview
        - devstack-no-tls-proxy
    periodic:
      jobs:
        - devstack-no-tls-proxy
+    periodic-weekly:
+      jobs:
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
@@ -4,7 +4,7 @@ from git source trees.
 Goals
 =====

-* To quickly build dev OpenStack environments in a clean Ubuntu or Fedora
+* To quickly build dev OpenStack environments in a clean Ubuntu or RockyLinux
  environment
 * To describe working configurations of OpenStack (which code branches
  work together?  what do config files look like for those branches?)
@@ -28,9 +28,9 @@ Versions
 The DevStack master branch generally points to trunk versions of OpenStack
 components.  For older, stable versions, look for branches named
 stable/[release] in the DevStack repo.  For example, you can do the
-following to create a Pike OpenStack cloud::
+following to create a Zed OpenStack cloud::

-    git checkout stable/pike
+    git checkout stable/zed
    ./stack.sh

 You can also pick specific OpenStack project releases by setting the appropriate
@@ -55,7 +55,7 @@ When the script finishes executing, you should be able to access OpenStack
 endpoints, like so:

 * Horizon: http://myhost/
-* Keystone: http://myhost/identity/v2.0/
+* Keystone: http://myhost/identity/v3/

 We also provide an environment file that you can use to interact with your
 cloud via CLI::
@@ -50,7 +50,6 @@ source $TOP_DIR/lib/placement
 source $TOP_DIR/lib/cinder
 source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/neutron
-source $TOP_DIR/lib/neutron-legacy

 set -o xtrace

@@ -4,8 +4,4 @@ Pygments
 docutils
 sphinx>=2.0.0,!=2.1.0 # BSD
 openstackdocstheme>=2.2.1 # Apache-2.0
-nwdiag
-blockdiag
-sphinxcontrib-blockdiag
-sphinxcontrib-nwdiag
 zuul-sphinx>=0.2.0
@@ -23,14 +23,14 @@

 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinx.ext.autodoc',
-               'zuul_sphinx',
-               'openstackdocstheme',
-               'sphinxcontrib.blockdiag',
-               'sphinxcontrib.nwdiag' ]
+extensions = [
+    'sphinx.ext.autodoc',
+    'zuul_sphinx',
+    'openstackdocstheme',
+]

 # openstackdocstheme options
-openstackdocs_repo_name = 'openstack-dev/devstack'
+openstackdocs_repo_name = 'openstack/devstack'
 openstackdocs_pdf_link = True
 openstackdocs_bug_project = 'devstack'
 openstackdocs_bug_tag = ''
@@ -181,6 +181,9 @@ values that most often need to be set.
 If the ``*_PASSWORD`` variables are not set here you will be prompted to
 enter values for them by ``stack.sh``.

+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.
+
 The network ranges must not overlap with any networks in use on the
 host. Overlap is not uncommon as RFC-1918 'private' ranges are commonly
 used for both the local networking and Nova's fixed and floating ranges.
@@ -279,7 +282,7 @@ number of days of old log files to keep.

 ::

-    LOGDAYS=1
+    LOGDAYS=2

 Some coloring is used during the DevStack runs to make it easier to
 see what is going on. This can be disabled with::
@@ -521,8 +524,8 @@ behavior:
 can be configured with any valid IPv6 prefix. The default values make
 use of an auto-generated ``IPV6_GLOBAL_ID`` to comply with RFC4193.

-Service Version
-~~~~~~~~~~~~~~~
+Service IP Version
+~~~~~~~~~~~~~~~~~~

 DevStack can enable service operation over either IPv4 or IPv6 by
 setting ``SERVICE_IP_VERSION`` to either ``SERVICE_IP_VERSION=4`` or
@@ -542,6 +545,27 @@ optionally be used to alter the default IPv6 address::

  HOST_IPV6=${some_local_ipv6_address}

+Tunnel IP Version
+~~~~~~~~~~~~~~~~~
+
+DevStack can enable tunnel operation over either IPv4 or IPv6 by
+setting ``TUNNEL_IP_VERSION`` to either ``TUNNEL_IP_VERSION=4`` or
+``TUNNEL_IP_VERSION=6`` respectively.
+
+When set to ``4`` Neutron will use an IPv4 address for tunnel endpoints,
+for example, ``HOST_IP``.
+
+When set to ``6`` Neutron will use an IPv6 address for tunnel endpoints,
+for example, ``HOST_IPV6``.
+
+The default value for this setting is ``4``.  Dual-mode support, for
+example ``4+6`` is not supported, as this value must match the address
+family of the local tunnel endpoint IP(v6) address.
+
+The value of ``TUNNEL_IP_VERSION`` has a direct relationship to the
+setting of ``TUNNEL_ENDPOINT_IP``, which will default to ``HOST_IP``
+when set to ``4``, and ``HOST_IPV6`` when set to ``6``.
+
 Multi-node setup
 ~~~~~~~~~~~~~~~~

@@ -615,7 +639,7 @@ tests can be run as follows:
 ::

    $ cd /opt/stack/tempest
-    $ tox -efull  tempest.scenario.test_network_basic_ops
+    $ tox -e smoke

 By default tempest is downloaded and the config file is generated, but the
 tempest package is not installed in the system's global site-packages (the
@@ -642,6 +666,41 @@ with ``VOLUME_BACKING_FILE_SIZE``.
  VOLUME_NAME_PREFIX="volume-"
  VOLUME_BACKING_FILE_SIZE=24G

+When running highly concurrent tests, the default per-project quotas
+for volumes, backups, or snapshots may be too small.  These can be
+adjusted by setting ``CINDER_QUOTA_VOLUMES``, ``CINDER_QUOTA_BACKUPS``,
+or ``CINDER_QUOTA_SNAPSHOTS`` to the desired value.  (The default for
+each is 10.)
+
+DevStack's Cinder LVM configuration module currently supports both iSCSI and
+NVMe connections, and we can choose which one to use with options
+``CINDER_TARGET_HELPER``, ``CINDER_TARGET_PROTOCOL``, ``CINDER_TARGET_PREFIX``,
+and ``CINDER_TARGET_PORT``.
+
+Defaults use iSCSI with the LIO target manager::
+
+  CINDER_TARGET_HELPER="lioadm"
+  CINDER_TARGET_PROTOCOL="iscsi"
+  CINDER_TARGET_PREFIX="iqn.2010-10.org.openstack:"
+  CINDER_TARGET_PORT=3260
+
+Additionally there are 3 supported transport protocols for NVMe,
+``nvmet_rdma``, ``nvmet_tcp``, and ``nvmet_fc``, and when the ``nvmet`` target
+is selected the protocol, prefix, and port defaults will change to more
+sensible defaults for NVMe::
+
+  CINDER_TARGET_HELPER="nvmet"
+  CINDER_TARGET_PROTOCOL="nvmet_rdma"
+  CINDER_TARGET_PREFIX="nvme-subsystem-1"
+  CINDER_TARGET_PORT=4420
+
+When selecting the RDMA transport protocol DevStack will create on Cinder nodes
+a Software RoCE device on top of the ``HOST_IP_IFACE`` and if it is not defined
+then on top of the interface with IP address ``HOST_IP`` or ``HOST_IPV6``.
+
+This Soft-RoCE device will always be created on the Nova compute side since we
+cannot tell beforehand whether there will be an RDMA connection or not.
+

 Keystone
 ~~~~~~~~
@@ -666,7 +725,6 @@ In RegionTwo:

    disable_service horizon
    KEYSTONE_SERVICE_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
-    KEYSTONE_AUTH_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
    REGION_NAME=RegionTwo
    KEYSTONE_REGION_NAME=RegionOne

@@ -679,17 +737,6 @@ KEYSTONE_REGION_NAME to specify the region of Keystone service.
 KEYSTONE_REGION_NAME has a default value the same as REGION_NAME thus we omit
 it in the configuration of RegionOne.

-Disabling Identity API v2
-+++++++++++++++++++++++++
-
-The Identity API v2 is deprecated as of Mitaka and it is recommended to only
-use the v3 API. It is possible to setup keystone without v2 API, by doing:
-
-::
-
-    ENABLE_IDENTITY_V2=False
-
-
 Glance
 ++++++

@@ -704,7 +751,7 @@ or at runtime via:

 ::

-    openstack --os-cloud devstack-system-admin registered limit update \
+    openstack --os-cloud devstack-system-admin registered limit set \
      --service glance --default-limit 5000 --region RegionOne image_size_total

 .. _arch-configuration:
@@ -42,8 +42,9 @@ Getting Your Patch Merged
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 All changes proposed to the Devstack require two ``Code-Review +2`` votes from
 Devstack core reviewers before one of the core reviewers can approve the patch
-by giving ``Workflow +1`` vote. One exception is for patches to unblock the gate
-which can be approved by single core reviewers.
+by giving ``Workflow +1`` vote. There are 2 exceptions, approving patches to
+unblock the gate and patches that do not relate to the Devstack's core logic,
+like for example old job cleanups, can be approved by single core reviewers.

 Project Team Lead Duties
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -20,6 +20,12 @@ provides consumption output when available memory is seen to be
 falling (i.e. processes are consuming memory).  It also provides
 output showing locked (unswappable) memory.

+file_tracker
+------------
+
+The ``file_tracker`` service periodically monitors the number of
+open files in the system.
+
 tcpdump
 -------

@@ -20,7 +20,7 @@ Walk through various setups used by stackers
   guides/neutron
   guides/devstack-with-nested-kvm
   guides/nova
-   guides/devstack-with-lbaas-v2
+   guides/devstack-with-octavia
   guides/devstack-with-ldap

 All-In-One Single VM
@@ -69,10 +69,10 @@ Nova and devstack

 Guide to working with nova features :doc:`Nova and devstack <guides/nova>`.

-Configure Load-Balancer Version 2
-----------------------------------
+Configure Octavia
+-----------------

-Guide on :doc:`Configure Load-Balancer Version 2 <guides/devstack-with-lbaas-v2>`.
+Guide on :doc:`Configure Octavia <guides/devstack-with-octavia>`.

 Deploying DevStack with LDAP
 ----------------------------
@@ -1,145 +0,0 @@
-Devstack with Octavia Load Balancing
-====================================
-
-Starting with the OpenStack Pike release, Octavia is now a standalone service
-providing load balancing services for OpenStack.
-
-This guide will show you how to create a devstack with `Octavia API`_ enabled.
-
-.. _Octavia API: https://docs.openstack.org/api-ref/load-balancer/v2/index.html
-
-Phase 1: Create DevStack + 2 nova instances
--------------------------------------------
-
-First, set up a vm of your choice with at least 8 GB RAM and 16 GB disk space,
-make sure it is updated. Install git and any other developer tools you find
-useful.
-
-Install devstack
-
-::
-
-    git clone https://opendev.org/openstack/devstack
-    cd devstack/tools
-    sudo ./create-stack-user.sh
-    cd ../..
-    sudo mv devstack /opt/stack
-    sudo chown -R stack.stack /opt/stack/devstack
-
-This will clone the current devstack code locally, then setup the "stack"
-account that devstack services will run under. Finally, it will move devstack
-into its default location in /opt/stack/devstack.
-
-Edit your ``/opt/stack/devstack/local.conf`` to look like
-
-::
-
-    [[local|localrc]]
-    enable_plugin octavia https://opendev.org/openstack/octavia
-    # If you are enabling horizon, include the octavia dashboard
-    # enable_plugin octavia-dashboard https://opendev.org/openstack/octavia-dashboard.git
-    # If you are enabling barbican for TLS offload in Octavia, include it here.
-    # enable_plugin barbican https://opendev.org/openstack/barbican
-
-    # ===== BEGIN localrc =====
-    DATABASE_PASSWORD=password
-    ADMIN_PASSWORD=password
-    SERVICE_PASSWORD=password
-    SERVICE_TOKEN=password
-    RABBIT_PASSWORD=password
-    # Enable Logging
-    LOGFILE=$DEST/logs/stack.sh.log
-    VERBOSE=True
-    LOG_COLOR=True
-    # Pre-requisite
-    ENABLED_SERVICES=rabbit,mysql,key
-    # Horizon - enable for the OpenStack web GUI
-    # ENABLED_SERVICES+=,horizon
-    # Nova
-    ENABLED_SERVICES+=,n-api,n-crt,n-cpu,n-cond,n-sch,n-api-meta,n-sproxy
-    ENABLED_SERVICES+=,placement-api,placement-client
-    # Glance
-    ENABLED_SERVICES+=,g-api
-    # Neutron
-    ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron
-    ENABLED_SERVICES+=,octavia,o-cw,o-hk,o-hm,o-api
-    # Cinder
-    ENABLED_SERVICES+=,c-api,c-vol,c-sch
-    # Tempest
-    ENABLED_SERVICES+=,tempest
-    # Barbican - Optionally used for TLS offload in Octavia
-    # ENABLED_SERVICES+=,barbican
-    # ===== END localrc =====
-
-Run stack.sh and do some sanity checks
-
-::
-
-    sudo su - stack
-    cd /opt/stack/devstack
-    ./stack.sh
-    . ./openrc
-
-    openstack network list  # should show public and private networks
-
-Create two nova instances that we can use as test http servers:
-
-::
-
-    #create nova instances on private network
-    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node1
-    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node2
-    openstack server list # should show the nova instances just created
-
-    #add secgroup rules to allow ssh etc..
-    openstack security group rule create default --protocol icmp
-    openstack security group rule create default --protocol tcp --dst-port 22:22
-    openstack security group rule create default --protocol tcp --dst-port 80:80
-
-Set up a simple web server on each of these instances. ssh into each instance (username 'cirros', password 'cubswin:)' or 'gocubsgo') and run
-
-::
-
-    MYIP=$(ifconfig eth0|grep 'inet addr'|awk -F: '{print $2}'| awk '{print $1}')
-    while true; do echo -e "HTTP/1.0 200 OK\r\n\r\nWelcome to $MYIP" | sudo nc -l -p 80 ; done&
-
-Phase 2: Create your load balancer
----------------------------------
-
-Make sure you have the 'openstack loadbalancer' commands:
-
-::
-
-    pip install python-octaviaclient
-
-Create your load balancer:
-
-::
-
-    openstack loadbalancer create --name lb1 --vip-subnet-id private-subnet
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer listener create --protocol HTTP --protocol-port 80 --name listener1 lb1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer pool create --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer healthmonitor create --delay 5 --timeout 2 --max-retries 1 --type HTTP pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer member create --subnet-id private-subnet --address <web server 1 address> --protocol-port 80 pool1
-    openstack loadbalancer show lb1  # Wait for the provisioning_status to be ACTIVE.
-    openstack loadbalancer member create --subnet-id private-subnet --address <web server 2 address> --protocol-port 80 pool1
-
-Please note: The <web server # address> fields are the IP addresses of the nova
-servers created in Phase 1.
-Also note, using the API directly you can do all of the above commands in one
-API call.
-
-Phase 3: Test your load balancer
--------------------------------
-
-::
-
-    openstack loadbalancer show lb1 # Note the vip_address
-    curl http://<vip_address>
-    curl http://<vip_address>
-
-This should show the "Welcome to <IP>" message from each member server.
@@ -1,3 +1,5 @@
+.. _kvm_nested_virt:
+
 =======================================================
 Configure DevStack with KVM-based Nested Virtualization
 =======================================================
@@ -0,0 +1,144 @@
+Devstack with Octavia Load Balancing
+====================================
+
+Starting with the OpenStack Pike release, Octavia is now a standalone service
+providing load balancing services for OpenStack.
+
+This guide will show you how to create a devstack with `Octavia API`_ enabled.
+
+.. _Octavia API: https://docs.openstack.org/api-ref/load-balancer/v2/index.html
+
+Phase 1: Create DevStack + 2 nova instances
+--------------------------------------------
+
+First, set up a VM of your choice with at least 8 GB RAM and 16 GB disk space,
+make sure it is updated. Install git and any other developer tools you find
+useful.
+
+Install devstack::
+
+    git clone https://opendev.org/openstack/devstack
+    cd devstack/tools
+    sudo ./create-stack-user.sh
+    cd ../..
+    sudo mv devstack /opt/stack
+    sudo chown -R stack.stack /opt/stack/devstack
+
+This will clone the current devstack code locally, then setup the "stack"
+account that devstack services will run under. Finally, it will move devstack
+into its default location in /opt/stack/devstack.
+
+Edit your ``/opt/stack/devstack/local.conf`` to look like::
+
+    [[local|localrc]]
+    # ===== BEGIN localrc =====
+    DATABASE_PASSWORD=password
+    ADMIN_PASSWORD=password
+    SERVICE_PASSWORD=password
+    SERVICE_TOKEN=password
+    RABBIT_PASSWORD=password
+    GIT_BASE=https://opendev.org
+    # Optional settings:
+    # OCTAVIA_AMP_BASE_OS=centos
+    # OCTAVIA_AMP_DISTRIBUTION_RELEASE_ID=9-stream
+    # OCTAVIA_AMP_IMAGE_SIZE=3
+    # OCTAVIA_LB_TOPOLOGY=ACTIVE_STANDBY
+    # OCTAVIA_ENABLE_AMPHORAV2_JOBBOARD=True
+    # LIBS_FROM_GIT+=octavia-lib,
+    # Enable Logging
+    LOGFILE=$DEST/logs/stack.sh.log
+    VERBOSE=True
+    LOG_COLOR=True
+    enable_service rabbit
+    enable_plugin neutron $GIT_BASE/openstack/neutron
+    # Octavia supports using QoS policies on the VIP port:
+    enable_service q-qos
+    enable_service placement-api placement-client
+    # Octavia services
+    enable_plugin octavia $GIT_BASE/openstack/octavia master
+    enable_plugin octavia-dashboard $GIT_BASE/openstack/octavia-dashboard
+    enable_plugin ovn-octavia-provider $GIT_BASE/openstack/ovn-octavia-provider
+    enable_plugin octavia-tempest-plugin $GIT_BASE/openstack/octavia-tempest-plugin
+    enable_service octavia o-api o-cw o-hm o-hk o-da
+    # If you are enabling barbican for TLS offload in Octavia, include it here.
+    # enable_plugin barbican $GIT_BASE/openstack/barbican
+    # enable_service barbican
+    # Cinder (optional)
+    disable_service c-api c-vol c-sch
+    # Tempest
+    enable_service tempest
+    # ===== END localrc =====
+
+.. note::
+    For best performance it is highly recommended to use KVM
+    virtualization instead of QEMU.
+    Also make sure nested virtualization is enabled as documented in
+    :ref:`the respective guide <kvm_nested_virt>`.
+    By adding ``LIBVIRT_CPU_MODE="host-passthrough"`` to your
+    ``local.conf`` you enable the guest VMs to make use of all features your
+    host's CPU provides.
+
+Run stack.sh and do some sanity checks::
+
+    sudo su - stack
+    cd /opt/stack/devstack
+    ./stack.sh
+    . ./openrc
+
+    openstack network list  # should show public and private networks
+
+Create two nova instances that we can use as test http servers::
+
+    # create nova instances on private network
+    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node1
+    openstack server create --image $(openstack image list | awk '/ cirros-.*-x86_64-.* / {print $2}') --flavor 1 --nic net-id=$(openstack network list | awk '/ private / {print $2}') node2
+    openstack server list # should show the nova instances just created
+
+    # add secgroup rules to allow ssh etc..
+    openstack security group rule create default --protocol icmp
+    openstack security group rule create default --protocol tcp --dst-port 22:22
+    openstack security group rule create default --protocol tcp --dst-port 80:80
+
+Set up a simple web server on each of these instances. One possibility is to use
+the `Golang test server`_ that is used by the Octavia project for CI testing
+as well.
+Copy the binary to your instances and start it as shown below
+(username 'cirros', password 'gocubsgo')::
+
+    INST_IP=<instance IP>
+    scp -O test_server.bin cirros@${INST_IP}:
+    ssh -f cirros@${INST_IP} ./test_server.bin -id ${INST_IP}
+
+When started this way the test server will respond to HTTP requests with
+its own IP.
+
+Phase 2: Create your load balancer
+----------------------------------
+
+Create your load balancer::
+
+    openstack loadbalancer create --wait --name lb1 --vip-subnet-id private-subnet
+    openstack loadbalancer listener create --wait --protocol HTTP --protocol-port 80 --name listener1 lb1
+    openstack loadbalancer pool create --wait --lb-algorithm ROUND_ROBIN --listener listener1 --protocol HTTP --name pool1
+    openstack loadbalancer healthmonitor create --wait --delay 5 --timeout 2 --max-retries 1 --type HTTP pool1
+    openstack loadbalancer member create --wait --subnet-id private-subnet --address <web server 1 address> --protocol-port 80 pool1
+    openstack loadbalancer member create --wait --subnet-id private-subnet --address <web server 2 address> --protocol-port 80 pool1
+
+Please note: The <web server # address> fields are the IP addresses of the nova
+servers created in Phase 1.
+Also note, using the API directly you can do all of the above commands in one
+API call.
+
+Phase 3: Test your load balancer
+--------------------------------
+
+::
+
+    openstack loadbalancer show lb1 # Note the vip_address
+    curl http://<vip_address>
+    curl http://<vip_address>
+
+This should show the "Welcome to <IP>" message from each member server.
+
+
+.. _Golang test server: https://opendev.org/openstack/octavia-tempest-plugin/src/branch/master/octavia_tempest_plugin/contrib/test_server
@@ -75,13 +75,21 @@ Otherwise create the stack user:

    useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+::
+
+    chmod +x /opt/stack
+
 This user will be making many changes to your system during installation
 and operation so it needs to have sudo privileges to root without a
 password:

 ::

-    echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 From here on use the ``stack`` user. **Logout** and **login** as the
 ``stack`` user.
@@ -169,7 +177,7 @@ machines, create a ``local.conf`` with:
    MYSQL_HOST=$SERVICE_HOST
    RABBIT_HOST=$SERVICE_HOST
    GLANCE_HOSTPORT=$SERVICE_HOST:9292
-    ENABLED_SERVICES=n-cpu,q-agt,c-vol,placement-client
+    ENABLED_SERVICES=n-cpu,c-vol,placement-client,ovn-controller,ovs-vswitchd,ovsdb-server,q-ovn-metadata-agent
    NOVA_VNC_ENABLED=True
    NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_lite.html"
    VNCSERVER_LISTEN=$HOST_IP
@@ -395,7 +403,7 @@ SSH keys need to be exchanged between each compute node:

 3. Verify that login via ssh works without a password::

-        ssh -i /root/.ssh/id_rsa.pub stack@DESTINATION
+        ssh -i /root/.ssh/id_rsa stack@DESTINATION

 In essence, this means that every compute node's root user's public RSA key
 must exist in every other compute node's stack user's authorized_keys file and
@@ -41,19 +41,8 @@ network and is on a shared subnet with other machines.  The
 `local.conf` exhibited here assumes that 1500 is a reasonable MTU to
 use on that network.

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                }
-        }
+.. image:: /assets/images/neutron-network-1.png
+   :alt: Network configuration for a single DevStack node


 DevStack Configuration
@@ -100,21 +89,8 @@ also want to do multinode testing and networking.
 Physical Network Setup
 ~~~~~~~~~~~~~~~~~~~~~~

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                        devstack-2 [ address = "172.18.161.7" ];
-                }
-        }
-
+.. image:: /assets/images/neutron-network-2.png
+   :alt: Network configuration for multiple DevStack nodes

 After DevStack installs and configures Neutron, traffic from guest VMs
 flows out of `devstack-2` (the compute node) and is encapsulated in a
@@ -222,8 +198,6 @@ connect OpenStack nodes (like `devstack-2`) together. This bridge is
 used so that project network traffic, using the VXLAN tunneling
 protocol, flows between each compute node where project instances run.

-
-
 DevStack Compute Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -268,30 +242,8 @@ to the neutron L3 service.
 Physical Network Setup
 ----------------------

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network provider_net {
-                        address = "203.0.113.0/24"
-                        router [ address = "203.0.113.1" ];
-                        controller;
-                        compute1;
-                        compute2;
-                }
-
-                network control_plane {
-                        router [ address = "10.0.0.1" ]
-                        address = "10.0.0.0/24"
-                        controller [ address = "10.0.0.2" ]
-                        compute1 [ address = "10.0.0.3" ]
-                        compute2 [ address = "10.0.0.4" ]
-                }
-        }
-
+.. image:: /assets/images/neutron-network-3.png
+   :alt: Network configuration for provider networks

 On a compute node, the first interface, eth0 is used for the OpenStack
 management (API, message bus, etc) as well as for ssh for an
@@ -122,7 +122,7 @@ when creating the server, for example:
 .. code-block:: shell

  $ openstack --os-compute-api-version 2.37 server create --flavor cirros256 \
-      --image cirros-0.3.5-x86_64-disk --nic none --wait test-server
+      --image cirros-0.6.3-x86_64-disk --nic none --wait test-server

 .. note:: ``--os-compute-api-version`` greater than or equal to 2.37 is
          required to use ``--nic=none``.
@@ -49,13 +49,21 @@ below)

    $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it will need
 to have sudo privileges:

 .. code-block:: console

    $ apt-get install sudo -y || yum install -y sudo
-    $ echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    $ echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 .. note:: On some systems you may need to use ``sudo visudo``.

@@ -98,6 +106,9 @@ do the following:
 -  Set the service password. This is used by the OpenStack services
   (Nova, Glance, etc) to authenticate with Keystone.

+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.
+
 ``local.conf`` should look something like this:

 .. code-block:: ini
@@ -37,10 +37,10 @@ Install Linux
 -------------

 Start with a clean and minimal install of a Linux system. DevStack
-attempts to support the two latest LTS releases of Ubuntu, the
-latest/current Fedora version, CentOS/RHEL 8 and OpenSUSE.
+attempts to support the two latest LTS releases of Ubuntu,
+Rocky Linux 9 and openEuler.

-If you do not have a preference, Ubuntu 20.04 (Focal Fossa) is the
+If you do not have a preference, Ubuntu 22.04 (Jammy) is the
 most tested, and will probably go the smoothest.

 Add Stack User (optional)
@@ -57,6 +57,14 @@ to run DevStack with

   $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it should
 have sudo privileges:

@@ -93,7 +101,10 @@ devstack git repo.
 This is the minimum required config to get started with DevStack.

 .. note:: There is a sample :download:`local.conf </assets/local.conf>` file
-    under the *samples* directory in the devstack repository.
+   under the *samples* directory in the devstack repository.
+
+.. warning:: Only use alphanumeric characters in your passwords, as some
+   services fail to work when using special characters.

 Start the install
 -----------------
@@ -102,7 +113,7 @@ Start the install

   $ ./stack.sh

-This will take a 15 - 20 minutes, largely depending on the speed of
+This will take 15 - 30 minutes, largely depending on the speed of
 your internet connection. Many git trees and packages will be
 installed during this process.

@@ -122,6 +133,8 @@ there.
 You can ``source openrc`` in your shell, and then use the
 ``openstack`` command line tool to manage your devstack.

+You can :ref:`create a VM and SSH into it <ssh>`.
+
 You can ``cd /opt/stack/tempest`` and run tempest tests that have
 been configured to work with your devstack.

@@ -68,7 +68,7 @@ Shared Guest Interface
 .. warning::

   This is not a recommended configuration. Because of interactions
-   between ovs and bridging, if you reboot your box with active
+   between OVS and bridging, if you reboot your box with active
   networking you may lose network connectivity to your system.

 If you need your guests accessible on the network, but only have 1
@@ -114,3 +114,125 @@ For IPv6, ``FIXED_RANGE_V6`` will default to the first /64 of the value of
 ``FIXED_RANGE_V6`` will just use the value of that directly.
 ``SUBNETPOOL_PREFIX_V6`` will just default to the value of
 ``IPV6_ADDRS_SAFE_TO_USE`` directly.
+
+.. _ssh:
+
+SSH access to instances
+=======================
+
+To validate connectivity, you can create an instance using the
+``$PRIVATE_NETWORK_NAME`` network (default: ``private``), create a floating IP
+using the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``), and attach
+this floating IP to the instance:
+
+.. code-block:: shell
+
+    openstack keypair create --public-key ~/.ssh/id_rsa.pub test-keypair
+    openstack server create --network private --key-name test-keypair ... test-server
+    fip_id=$(openstack floating ip create public -f value -c id)
+    openstack server add floating ip test-server ${fip_id}
+
+Once done, ensure you have enabled SSH and ICMP (ping) access for the security
+group used for the instance. You can either create a custom security group and
+specify it when creating the instance or add it after creation, or you can
+modify the ``default`` security group created by default for each project.
+Let's do the latter:
+
+.. code-block:: shell
+
+    openstack security group rule create --proto icmp --dst-port 0 default
+    openstack security group rule create --proto tcp --dst-port 22 default
+
+Finally, SSH into the instance. If you used the Cirros instance uploaded by
+default, then you can run the following:
+
+.. code-block:: shell
+
+    openstack server ssh test-server -- -l cirros
+
+This will connect using the ``cirros`` user and the keypair you configured when
+creating the instance.
+
+Remote SSH access to instances
+==============================
+
+You can also SSH to created instances on your DevStack host from other hosts.
+This can be helpful if you are e.g. deploying DevStack in a VM on an existing
+cloud and wish to do development on your local machine. There are a few ways to
+do this.
+
+.. rubric:: Configure instances to be locally accessible
+
+The most obvious way is to configure guests to be locally accessible, as
+described `above <Locally Accessible Guests>`__. This has the advantage of
+requiring no further effort on the client. However, it is more involved and
+requires either support from your cloud or some inadvisable workarounds.
+
+.. rubric:: Use your DevStack host as a jump host
+
+You can choose to use your DevStack host as a jump host. To SSH to a instance
+this way, pass the standard ``-J`` option to the ``openstack ssh`` / ``ssh``
+command. For example:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -J username@devstack-host
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+This can also be configured via your ``~/.ssh/config`` file, making it rather
+effortless. However, it only allows SSH access. If you want to access e.g. a
+web application on the instance, you will need to configure an SSH tunnel and
+forward select ports using the ``-L`` option. For example, to forward HTTP
+traffic:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -L 8080:username@devstack-host:80
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+As you can imagine, this can quickly get out of hand, particularly for more
+complex guest applications with multiple ports.
+
+.. rubric:: Use a proxy or VPN tool
+
+You can use a proxy or VPN tool to enable tunneling for the floating IP
+address range of the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``)
+defined by ``$FLOATING_RANGE`` (default: ``172.24.4.0/24``). There are many
+such tools available to do this. For example, we could use a useful utility
+called `shuttle`__. To enable tunneling using ``shuttle``, first ensure you
+have allowed SSH and HTTP(S) traffic to your DevStack host. Allowing HTTP(S)
+traffic is necessary so you can use the OpenStack APIs remotely. How you do
+this will depend on where your DevStack host is running. Once this is done,
+install ``sshuttle`` on your localhost:
+
+.. code-block:: bash
+
+    sudo apt-get install sshuttle || yum install sshuttle
+
+Finally, start ``sshuttle`` on your localhost using the floating IP address
+range. For example, assuming you are using the default value for
+``$FLOATING_RANGE``, you can do:
+
+.. code-block:: bash
+
+    sshuttle -r username@devstack-host 172.24.4.0/24
+
+(where ``username`` and ``devstack-host`` are the username and hostname of your
+DevStack host).
+
+You should now be able to create an instance and SSH into it:
+
+.. code-block:: bash
+
+    openstack server ssh test-server -- -l cirros
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`)
+
+.. __: https://github.com/sshuttle/sshuttle
@@ -23,13 +23,12 @@ strategy to include the latest Ubuntu release and the latest RHEL
 release.*

 -  Ubuntu: current LTS release plus current development release
-  Fedora: current release plus previous release
-  RHEL/CentOS: current major release
+-  RHEL/CentOS/RockyLinux: current major release
 -  Other OS platforms may continue to be included but the maintenance of
   those platforms shall not be assumed simply due to their presence.
   Having a listed point-of-contact for each additional OS will greatly
   increase its chance of being well-maintained.
-  Patches for Ubuntu and/or Fedora will not be held up due to
+-  Patches for Ubuntu and/or RockyLinux will not be held up due to
   side-effects on other OS platforms.

 Databases
@@ -24,14 +24,10 @@ official OpenStack projects.
 ======================================== ===
 Plugin Name                              URL
 ======================================== ===
-inspur/venus                             `https://opendev.org/inspur/venus <https://opendev.org/inspur/venus>`__
-inspur/venus-dashboard                   `https://opendev.org/inspur/venus-dashboard <https://opendev.org/inspur/venus-dashboard>`__
 openstack/aodh                           `https://opendev.org/openstack/aodh <https://opendev.org/openstack/aodh>`__
 openstack/barbican                       `https://opendev.org/openstack/barbican <https://opendev.org/openstack/barbican>`__
 openstack/blazar                         `https://opendev.org/openstack/blazar <https://opendev.org/openstack/blazar>`__
 openstack/ceilometer                     `https://opendev.org/openstack/ceilometer <https://opendev.org/openstack/ceilometer>`__
-openstack/ceilometer-powervm             `https://opendev.org/openstack/ceilometer-powervm <https://opendev.org/openstack/ceilometer-powervm>`__
-openstack/cinderlib                      `https://opendev.org/openstack/cinderlib <https://opendev.org/openstack/cinderlib>`__
 openstack/cloudkitty                     `https://opendev.org/openstack/cloudkitty <https://opendev.org/openstack/cloudkitty>`__
 openstack/cyborg                         `https://opendev.org/openstack/cyborg <https://opendev.org/openstack/cyborg>`__
 openstack/designate                      `https://opendev.org/openstack/designate <https://opendev.org/openstack/designate>`__
@@ -41,7 +37,6 @@ openstack/devstack-plugin-container      `https://opendev.org/openstack/devstack
 openstack/devstack-plugin-kafka          `https://opendev.org/openstack/devstack-plugin-kafka <https://opendev.org/openstack/devstack-plugin-kafka>`__
 openstack/devstack-plugin-nfs            `https://opendev.org/openstack/devstack-plugin-nfs <https://opendev.org/openstack/devstack-plugin-nfs>`__
 openstack/devstack-plugin-open-cas       `https://opendev.org/openstack/devstack-plugin-open-cas <https://opendev.org/openstack/devstack-plugin-open-cas>`__
-openstack/ec2-api                        `https://opendev.org/openstack/ec2-api <https://opendev.org/openstack/ec2-api>`__
 openstack/freezer                        `https://opendev.org/openstack/freezer <https://opendev.org/openstack/freezer>`__
 openstack/freezer-api                    `https://opendev.org/openstack/freezer-api <https://opendev.org/openstack/freezer-api>`__
 openstack/freezer-tempest-plugin         `https://opendev.org/openstack/freezer-tempest-plugin <https://opendev.org/openstack/freezer-tempest-plugin>`__
@@ -53,9 +48,7 @@ openstack/ironic-inspector               `https://opendev.org/openstack/ironic-i
 openstack/ironic-prometheus-exporter     `https://opendev.org/openstack/ironic-prometheus-exporter <https://opendev.org/openstack/ironic-prometheus-exporter>`__
 openstack/ironic-ui                      `https://opendev.org/openstack/ironic-ui <https://opendev.org/openstack/ironic-ui>`__
 openstack/keystone                       `https://opendev.org/openstack/keystone <https://opendev.org/openstack/keystone>`__
-openstack/kuryr-kubernetes               `https://opendev.org/openstack/kuryr-kubernetes <https://opendev.org/openstack/kuryr-kubernetes>`__
 openstack/kuryr-libnetwork               `https://opendev.org/openstack/kuryr-libnetwork <https://opendev.org/openstack/kuryr-libnetwork>`__
-openstack/kuryr-tempest-plugin           `https://opendev.org/openstack/kuryr-tempest-plugin <https://opendev.org/openstack/kuryr-tempest-plugin>`__
 openstack/magnum                         `https://opendev.org/openstack/magnum <https://opendev.org/openstack/magnum>`__
 openstack/magnum-ui                      `https://opendev.org/openstack/magnum-ui <https://opendev.org/openstack/magnum-ui>`__
 openstack/manila                         `https://opendev.org/openstack/manila <https://opendev.org/openstack/manila>`__
@@ -66,41 +59,37 @@ openstack/mistral                        `https://opendev.org/openstack/mistral
 openstack/monasca-api                    `https://opendev.org/openstack/monasca-api <https://opendev.org/openstack/monasca-api>`__
 openstack/monasca-events-api             `https://opendev.org/openstack/monasca-events-api <https://opendev.org/openstack/monasca-events-api>`__
 openstack/monasca-tempest-plugin         `https://opendev.org/openstack/monasca-tempest-plugin <https://opendev.org/openstack/monasca-tempest-plugin>`__
-openstack/murano                         `https://opendev.org/openstack/murano <https://opendev.org/openstack/murano>`__
 openstack/networking-bagpipe             `https://opendev.org/openstack/networking-bagpipe <https://opendev.org/openstack/networking-bagpipe>`__
 openstack/networking-baremetal           `https://opendev.org/openstack/networking-baremetal <https://opendev.org/openstack/networking-baremetal>`__
 openstack/networking-bgpvpn              `https://opendev.org/openstack/networking-bgpvpn <https://opendev.org/openstack/networking-bgpvpn>`__
 openstack/networking-generic-switch      `https://opendev.org/openstack/networking-generic-switch <https://opendev.org/openstack/networking-generic-switch>`__
-openstack/networking-hyperv              `https://opendev.org/openstack/networking-hyperv <https://opendev.org/openstack/networking-hyperv>`__
-openstack/networking-odl                 `https://opendev.org/openstack/networking-odl <https://opendev.org/openstack/networking-odl>`__
-openstack/networking-powervm             `https://opendev.org/openstack/networking-powervm <https://opendev.org/openstack/networking-powervm>`__
 openstack/networking-sfc                 `https://opendev.org/openstack/networking-sfc <https://opendev.org/openstack/networking-sfc>`__
 openstack/neutron                        `https://opendev.org/openstack/neutron <https://opendev.org/openstack/neutron>`__
 openstack/neutron-dynamic-routing        `https://opendev.org/openstack/neutron-dynamic-routing <https://opendev.org/openstack/neutron-dynamic-routing>`__
+openstack/neutron-fwaas                  `https://opendev.org/openstack/neutron-fwaas <https://opendev.org/openstack/neutron-fwaas>`__
+openstack/neutron-fwaas-dashboard        `https://opendev.org/openstack/neutron-fwaas-dashboard <https://opendev.org/openstack/neutron-fwaas-dashboard>`__
 openstack/neutron-tempest-plugin         `https://opendev.org/openstack/neutron-tempest-plugin <https://opendev.org/openstack/neutron-tempest-plugin>`__
 openstack/neutron-vpnaas                 `https://opendev.org/openstack/neutron-vpnaas <https://opendev.org/openstack/neutron-vpnaas>`__
 openstack/neutron-vpnaas-dashboard       `https://opendev.org/openstack/neutron-vpnaas-dashboard <https://opendev.org/openstack/neutron-vpnaas-dashboard>`__
-openstack/nova-powervm                   `https://opendev.org/openstack/nova-powervm <https://opendev.org/openstack/nova-powervm>`__
+openstack/nova                           `https://opendev.org/openstack/nova <https://opendev.org/openstack/nova>`__
 openstack/octavia                        `https://opendev.org/openstack/octavia <https://opendev.org/openstack/octavia>`__
 openstack/octavia-dashboard              `https://opendev.org/openstack/octavia-dashboard <https://opendev.org/openstack/octavia-dashboard>`__
 openstack/octavia-tempest-plugin         `https://opendev.org/openstack/octavia-tempest-plugin <https://opendev.org/openstack/octavia-tempest-plugin>`__
 openstack/openstacksdk                   `https://opendev.org/openstack/openstacksdk <https://opendev.org/openstack/openstacksdk>`__
 openstack/osprofiler                     `https://opendev.org/openstack/osprofiler <https://opendev.org/openstack/osprofiler>`__
-openstack/oswin-tempest-plugin           `https://opendev.org/openstack/oswin-tempest-plugin <https://opendev.org/openstack/oswin-tempest-plugin>`__
+openstack/ovn-bgp-agent                  `https://opendev.org/openstack/ovn-bgp-agent <https://opendev.org/openstack/ovn-bgp-agent>`__
 openstack/ovn-octavia-provider           `https://opendev.org/openstack/ovn-octavia-provider <https://opendev.org/openstack/ovn-octavia-provider>`__
-openstack/patrole                        `https://opendev.org/openstack/patrole <https://opendev.org/openstack/patrole>`__
 openstack/rally-openstack                `https://opendev.org/openstack/rally-openstack <https://opendev.org/openstack/rally-openstack>`__
-openstack/sahara                         `https://opendev.org/openstack/sahara <https://opendev.org/openstack/sahara>`__
-openstack/sahara-dashboard               `https://opendev.org/openstack/sahara-dashboard <https://opendev.org/openstack/sahara-dashboard>`__
-openstack/senlin                         `https://opendev.org/openstack/senlin <https://opendev.org/openstack/senlin>`__
 openstack/shade                          `https://opendev.org/openstack/shade <https://opendev.org/openstack/shade>`__
-openstack/solum                          `https://opendev.org/openstack/solum <https://opendev.org/openstack/solum>`__
+openstack/skyline-apiserver              `https://opendev.org/openstack/skyline-apiserver <https://opendev.org/openstack/skyline-apiserver>`__
 openstack/storlets                       `https://opendev.org/openstack/storlets <https://opendev.org/openstack/storlets>`__
 openstack/tacker                         `https://opendev.org/openstack/tacker <https://opendev.org/openstack/tacker>`__
 openstack/tap-as-a-service               `https://opendev.org/openstack/tap-as-a-service <https://opendev.org/openstack/tap-as-a-service>`__
 openstack/telemetry-tempest-plugin       `https://opendev.org/openstack/telemetry-tempest-plugin <https://opendev.org/openstack/telemetry-tempest-plugin>`__
 openstack/trove                          `https://opendev.org/openstack/trove <https://opendev.org/openstack/trove>`__
 openstack/trove-dashboard                `https://opendev.org/openstack/trove-dashboard <https://opendev.org/openstack/trove-dashboard>`__
+openstack/venus                          `https://opendev.org/openstack/venus <https://opendev.org/openstack/venus>`__
+openstack/venus-dashboard                `https://opendev.org/openstack/venus-dashboard <https://opendev.org/openstack/venus-dashboard>`__
 openstack/vitrage                        `https://opendev.org/openstack/vitrage <https://opendev.org/openstack/vitrage>`__
 openstack/vitrage-dashboard              `https://opendev.org/openstack/vitrage-dashboard <https://opendev.org/openstack/vitrage-dashboard>`__
 openstack/vitrage-tempest-plugin         `https://opendev.org/openstack/vitrage-tempest-plugin <https://opendev.org/openstack/vitrage-tempest-plugin>`__
@@ -112,7 +101,6 @@ openstack/zaqar-ui                       `https://opendev.org/openstack/zaqar-ui
 openstack/zun                            `https://opendev.org/openstack/zun <https://opendev.org/openstack/zun>`__
 openstack/zun-ui                         `https://opendev.org/openstack/zun-ui <https://opendev.org/openstack/zun-ui>`__
 performa/os-faults                       `https://opendev.org/performa/os-faults <https://opendev.org/performa/os-faults>`__
-skyline/skyline-apiserver                `https://opendev.org/skyline/skyline-apiserver <https://opendev.org/skyline/skyline-apiserver>`__
 starlingx/config                         `https://opendev.org/starlingx/config <https://opendev.org/starlingx/config>`__
 starlingx/fault                          `https://opendev.org/starlingx/fault <https://opendev.org/starlingx/fault>`__
 starlingx/ha                             `https://opendev.org/starlingx/ha <https://opendev.org/starlingx/ha>`__
@@ -186,6 +174,7 @@ x/trio2o                                 `https://opendev.org/x/trio2o <https://
 x/valet                                  `https://opendev.org/x/valet <https://opendev.org/x/valet>`__
 x/vmware-nsx                             `https://opendev.org/x/vmware-nsx <https://opendev.org/x/vmware-nsx>`__
 x/vmware-vspc                            `https://opendev.org/x/vmware-vspc <https://opendev.org/x/vmware-vspc>`__
+x/whitebox-neutron-tempest-plugin        `https://opendev.org/x/whitebox-neutron-tempest-plugin <https://opendev.org/x/whitebox-neutron-tempest-plugin>`__
 ======================================== ===


@@ -238,14 +238,11 @@ package dependencies, packages may be listed at the following
 locations in the top-level of the plugin repository:

 - ``./devstack/files/debs/$plugin_name`` - Packages to install when running
-  on Ubuntu, Debian or Linux Mint.
+  on Ubuntu or Debian.

 - ``./devstack/files/rpms/$plugin_name`` - Packages to install when running
  on Red Hat, Fedora, or CentOS.

- ``./devstack/files/rpms-suse/$plugin_name`` - Packages to install when
-  running on SUSE Linux or openSUSE.
-
 Although there a no plans to remove this method of installing
 packages, plugins should consider it deprecated for ``bindep`` support
 described below.
@@ -0,0 +1,25 @@
+=======
+Tempest
+=======
+
+`Tempest`_ is the OpenStack Integration test suite. It is installed by default
+and is used to provide integration testing for many of the OpenStack services.
+Just like DevStack itself, it is possible to extend Tempest with plugins. In
+fact, many Tempest plugin packages also include DevStack plugin to do things
+like pre-create required static resources.
+
+The `Tempest documentation <Tempest>`_ provides a thorough guide to using
+Tempest. However, if you simply wish to run the standard set of Tempest tests
+against an existing deployment, you can do the following:
+
+.. code-block:: shell
+
+    cd /opt/stack/tempest
+    /opt/stack/data/venv/bin/tempest run ...
+
+The above assumes you have installed DevStack in the default location
+(configured via the ``DEST`` configuration variable) and have enabled
+virtualenv-based installation in the standard location (configured via the
+``USE_VENV`` and ``VENV_DEST`` configuration variables, respectively).
+
+.. _Tempest: https://docs.openstack.org/tempest/latest/
@@ -39,4 +39,5 @@
    CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined
 </VirtualHost>

+%WSGIPYTHONHOME%
 WSGISocketPrefix /var/run/%APACHE_NAME%
@@ -1,5 +1,4 @@
 Listen %PUBLICPORT%
-Listen %ADMINPORT%
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" keystone_combined

 <Directory %KEYSTONE_BIN%>
@@ -20,24 +19,11 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
    %SSLKEYFILE%
 </VirtualHost>

-<VirtualHost *:%ADMINPORT%>
-    WSGIDaemonProcess keystone-admin processes=3 threads=1 user=%USER% display-name=%{GROUP} %VIRTUALENV%
-    WSGIProcessGroup keystone-admin
-    WSGIScriptAlias / %KEYSTONE_BIN%/keystone-wsgi-admin
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-    ErrorLogFormat "%M"
-    ErrorLog /var/log/%APACHE_NAME%/keystone.log
-    CustomLog /var/log/%APACHE_NAME%/keystone_access.log keystone_combined
-    %SSLENGINE%
-    %SSLCERTFILE%
-    %SSLKEYFILE%
-</VirtualHost>
-
 %SSLLISTEN%<VirtualHost *:443>
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
 %SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%    SSLProtocol -all +TLSv1.3 +TLSv1.2
 %SSLLISTEN%</VirtualHost>

 Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
@@ -49,13 +35,3 @@ Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
    WSGIApplicationGroup %{GLOBAL}
    WSGIPassAuthorization On
 </Location>
-
-Alias /identity_admin %KEYSTONE_BIN%/keystone-wsgi-admin
-<Location /identity_admin>
-    SetHandler wsgi-script
-    Options +ExecCGI
-
-    WSGIProcessGroup keystone-admin
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-</Location>
@@ -24,6 +24,7 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
 %SSLLISTEN%    %SSLKEYFILE%
+%SSLLISTEN%    SSLProtocol -all +TLSv1.3 +TLSv1.2
 %SSLLISTEN%</VirtualHost>

 Alias /networking %NEUTRON_BIN%/neutron-api
@@ -1,7 +1,5 @@
 conntrack
 curl
-dnsmasq-base
-dnsmasq-utils # for dhcp_release
 ebtables
 genisoimage # required for config_drive
 iptables
@@ -2,5 +2,6 @@ curl
 liberasurecode-dev
 make
 memcached
+rsync
 sqlite3
 xfsprogs
@@ -1,3 +0,0 @@
-enable-tftp
-tftp-root=/tftpboot
-dhcp-boot=pxelinux.0
@@ -1,4 +1,4 @@
-dn: olcDatabase={${LDAP_OLCDB_NUMBER}}hdb,cn=config
+dn: olcDatabase={${LDAP_OLCDB_NUMBER}}${LDAP_OLCDB_TYPE},cn=config
 changetype: modify
 replace: olcSuffix
 olcSuffix: ${BASE_DN}
@@ -0,0 +1,16 @@
+[Unit]
+Description=Activate LVM backing file %BACKING_FILE%
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+ExecStart=/sbin/losetup --find --show %DIRECTIO% %BACKING_FILE%
+ExecStop=/bin/sh -c '/sbin/losetup -d $$(/sbin/losetup --associated %BACKING_FILE% -O NAME -n)'
+RemainAfterExit=yes
+Type=oneshot
+
+[Install]
+WantedBy=local-fs.target
+Also=systemd-udev-settle.service
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import os.path
+import json
+
+server_address = "/tmp/openstack.sock"
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+try:
+    sock.connect(server_address)
+except socket.error as msg:
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+
+    char = sock.recv(1)
+    if len(char) == 0:
+        print("Unexpected end of file", file=sys.stderr)
+        sys.exit(1)
+
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+        if len(char) == 0:
+            print("Unexpected end of file", file=sys.stderr)
+            sys.exit(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+try:
+    env = {}
+    passenv = ["CINDER_VERSION",
+               "OS_AUTH_URL",
+               "OS_NO_CACHE",
+               "OS_PASSWORD",
+               "OS_PROJECT_NAME",
+               "OS_REGION_NAME",
+               "OS_TENANT_NAME",
+               "OS_USERNAME",
+               "OS_VOLUME_API_VERSION",
+               "OS_CLOUD"]
+    for name in passenv:
+        if name in os.environ:
+            env[name] = os.environ[name]
+
+    cmd = {
+        "app": os.path.basename(sys.argv[0]),
+        "env": env,
+        "argv": sys.argv[1:]
+    }
+    try:
+        image_idx = sys.argv.index('image')
+        create_idx = sys.argv.index('create')
+        missing_file = image_idx < create_idx and \
+                not any(x.startswith('--file') for x in sys.argv)
+    except ValueError:
+        missing_file = False
+
+    if missing_file:
+        # This means we were called with an image create command, but were
+        # not provided a --file option. That likely means we're being passed
+        # the image data to stdin, which won't work because we do not proxy
+        # stdin to the server. So, we just reject the operation and ask the
+        # caller to provide the file with --file instead.
+        # We've already connected to the server, we need to send it some dummy
+        # data so it doesn't wait forever.
+        send(sock, {})
+        print('Image create without --file is not allowed in server mode',
+              file=sys.stderr)
+        sys.exit(1)
+    else:
+        send(sock, cmd)
+
+    doc = recv(sock)
+    if doc["stdout"] != b'':
+        print(doc["stdout"], end='')
+    if doc["stderr"] != b'':
+        print(doc["stderr"], file=sys.stderr)
+    sys.exit(doc["status"])
+finally:
+    sock.close()
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import json
+
+from openstackclient import shell as osc_shell
+from io import StringIO
+
+server_address = "/tmp/openstack.sock"
+
+try:
+    os.unlink(server_address)
+except OSError:
+    if os.path.exists(server_address):
+        raise
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+print('starting up on %s' % server_address, file=sys.stderr)
+sock.bind(server_address)
+
+# Listen for incoming connections
+sock.listen(1)
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+    char = sock.recv(1)
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+while True:
+    csock, client_address = sock.accept()
+    try:
+        doc = recv(csock)
+
+        print("%s %s" % (doc["app"], doc["argv"]), file=sys.stderr)
+        oldenv = {}
+        for name in doc["env"].keys():
+            oldenv[name] = os.environ.get(name, None)
+            os.environ[name] = doc["env"][name]
+
+        try:
+            old_stdout = sys.stdout
+            old_stderr = sys.stderr
+            my_stdout = sys.stdout = StringIO()
+            my_stderr = sys.stderr = StringIO()
+
+            class Exit(BaseException):
+                def __init__(self, status):
+                    self.status = status
+
+            def noexit(stat):
+                raise Exit(stat)
+
+            sys.exit = noexit
+
+            if doc["app"] == "openstack":
+                sh = osc_shell.OpenStackShell()
+                ret = sh.run(doc["argv"])
+            else:
+                print("Unknown application %s" % doc["app"], file=sys.stderr)
+                ret = 1
+        except Exit as e:
+            ret = e.status
+        finally:
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+
+            for name in oldenv.keys():
+                if oldenv[name] is None:
+                    del os.environ[name]
+                else:
+                    os.environ[name] = oldenv[name]
+
+        send(csock, {
+            "stdout": my_stdout.getvalue(),
+            "stderr": my_stderr.getvalue(),
+            "status": ret,
+        })
+
+    except BaseException as e:
+        print(e, file=sys.stderr)
+    finally:
+        csock.close()
@@ -1 +0,0 @@
-dnsmasq
@@ -1,3 +0,0 @@
-ceph    # NOPRIME
-lsb
-xfsprogs
@@ -1,3 +0,0 @@
-lvm2
-qemu-tools
-tgt # NOPRIME
@@ -1 +0,0 @@
-dstat
@@ -1,34 +0,0 @@
-apache2
-apache2-devel
-bc
-ca-certificates-mozilla
-curl
-gawk
-gcc
-gcc-c++
-git-core
-graphviz # docs
-iputils
-libffi-devel  # pyOpenSSL
-libjpeg8-devel # Pillow 3.0.0
-libopenssl-devel # to rebuild pyOpenSSL if needed
-libxslt-devel  # lxml
-lsof # useful when debugging
-make
-net-tools
-openssh
-openssl
-pcre-devel # python-pcre
-postgresql-devel  # psycopg2
-psmisc
-python3-systemd
-python-cmd2 # dist:opensuse-12.3
-python-devel  # pyOpenSSL
-python-xml
-tar
-tcpdump
-unzip
-util-linux
-wget
-which
-zlib-devel
@@ -1,2 +0,0 @@
-apache2-mod_wsgi  # NOPRIME
-apache2  # NOPRIME
@@ -1,4 +0,0 @@
-cyrus-sasl-devel
-memcached
-openldap2-devel
-sqlite3
@@ -1,3 +0,0 @@
-openldap2
-openldap2-client
-python-ldap
@@ -1 +0,0 @@
-python-dateutil
@@ -1,10 +0,0 @@
-cdrkit-cdrtools-compat # dist:sle12
-cryptsetup
-dosfstools
-libosinfo
-lvm2
-mkisofs # not:sle12
-open-iscsi
-sg3_utils
-# Stuff for diablo volumes
-sysfsutils
@@ -1 +0,0 @@
-ipset
@@ -1,12 +0,0 @@
-acl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
-ebtables
-haproxy # to serve as metadata proxy inside router/dhcp namespaces
-iptables
-iputils
-rabbitmq-server # NOPRIME
-radvd # NOPRIME
-sqlite3
-sudo
-vlan
@@ -1,2 +0,0 @@
-conntrack-tools
-keepalived
@@ -1,23 +0,0 @@
-cdrkit-cdrtools-compat # dist:sle12
-conntrack-tools
-curl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
-ebtables
-iptables
-iputils
-kpartx
-kvm # NOPRIME
-libvirt # NOPRIME
-libvirt-python # NOPRIME
-# mkisofs is required for config_drive
-mkisofs # not:sle12
-parted
-polkit
-# qemu as fallback if kvm cannot be used
-qemu # NOPRIME
-rabbitmq-server # NOPRIME
-socat
-sqlite3
-sudo
-vlan
@@ -1,3 +0,0 @@
-
-openvswitch
-openvswitch-switch
@@ -1,2 +0,0 @@
-lsscsi
-open-iscsi
@@ -1 +0,0 @@
-neutron-agent
@@ -1 +0,0 @@
-neutron-l3
@@ -1,6 +0,0 @@
-curl
-liberasurecode-devel
-memcached
-sqlite3
-xfsprogs
-xinetd
@@ -1,3 +1,3 @@
 ceph    # NOPRIME
-redhat-lsb-core
+redhat-lsb-core # not:rhel9,openEuler-22.03
 xfsprogs
@@ -6,9 +6,11 @@ gcc
 gcc-c++
 gettext  # used for compiling message catalogs
 git-core
+glibc-langpack-en # dist:rhel9
 graphviz # needed only for docs
 httpd
 httpd-devel
+iptables-nft # dist:rhel9
 iptables-services
 java-1.8.0-openjdk-headless
 libffi-devel
@@ -16,6 +18,7 @@ libjpeg-turbo-devel # Pillow 3.0.0
 libxml2-devel # lxml
 libxslt-devel # lxml
 libyaml-devel
+mod_ssl # required for tls-proxy on centos 9 stream computes
 net-tools
 openssh-server
 openssl
@@ -25,9 +28,9 @@ pkgconfig
 postgresql-devel  # psycopg2
 psmisc
 python3-devel
-python3-pip
+python3-pip # not:openEuler-22.03
 python3-systemd
-redhat-rpm-config # missing dep for gcc hardening flags, see rhbz#1217376
+redhat-rpm-config # not:openEuler-22.03  missing dep for gcc hardening flags, see rhbz#1217376
 tar
 tcpdump
 unzip
@@ -1,9 +1,10 @@
 cryptsetup
 dosfstools
-genisoimage
+genisoimage # not:rhel9
 iscsi-initiator-utils
 libosinfo
 lvm2
 sg3_utils
 # Stuff for diablo volumes
 sysfsutils
+xorriso # not:rhel8
@@ -1,15 +1,14 @@
 conntrack-tools
 curl
-dnsmasq # for q-dhcp
-dnsmasq-utils # for dhcp_release
 ebtables
-genisoimage # required for config_drive
+genisoimage # not:rhel9 required for config_drive
 iptables
 iputils
-kernel-modules
+kernel-modules # not:openEuler-22.03
 kpartx
 parted
 polkit
 rabbitmq-server # NOPRIME
 sqlite
 sudo
+xorriso # not:rhel8
@@ -4,4 +4,4 @@ memcached
 rsync-daemon
 sqlite
 xfsprogs
-xinetd # not:f34
+xinetd # not:f36,rhel9
@@ -118,7 +118,7 @@ function _upload_image {
        useimport="--import"
    fi

-    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties < "${image}"
+    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties --file $(readlink -f "${image}")
 }

 # Retrieve an image from a URL and upload into Glance.
@@ -133,17 +133,28 @@ function upload_image {

    local image image_fname image_name

+    local max_attempts=5
+
    # Create a directory for the downloaded image tarballs.
    mkdir -p $FILES/images
    image_fname=`basename "$image_url"`
    if [[ $image_url != file* ]]; then
        # Downloads the image (uec ami+akistyle), then extracts it.
        if [[ ! -f $FILES/$image_fname || "$(stat -c "%s" $FILES/$image_fname)" = "0" ]]; then
-            wget --progress=dot:giga -c $image_url -O $FILES/$image_fname
-            if [[ $? -ne 0 ]]; then
-                echo "Not found: $image_url"
-                return
-            fi
+            for attempt in `seq $max_attempts`; do
+                local rc=0
+                wget --progress=dot:giga -c $image_url -O $FILES/$image_fname || rc=$?
+                if [[ $rc -ne 0 ]]; then
+                    if [[ "$attempt" -eq "$max_attempts" ]]; then
+                        echo "Not found: $image_url"
+                        return
+                    fi
+                    echo "Download failed, retrying in $attempt second, attempt: $attempt"
+                    sleep $attempt
+                else
+                    break
+                fi
+            done
        fi
        image="$FILES/${image_fname}"
    else
@@ -414,10 +425,10 @@ function upload_image {
        # kernel for use when uploading the root filesystem.
        local kernel_id="" ramdisk_id="";
        if [ -n "$kernel" ]; then
-            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki < "$kernel" | grep ' id ' | get_field 2)
+            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki --file $(readlink -f "$kernel") -f value -c id)
        fi
        if [ -n "$ramdisk" ]; then
-            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari < "$ramdisk" | grep ' id ' | get_field 2)
+            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari --file $(readlink -f "$ramdisk") -f value -c id)
        fi
        _upload_image "${image_name%.img}" ami ami "$image" ${kernel_id:+ kernel_id=$kernel_id} ${ramdisk_id:+ ramdisk_id=$ramdisk_id} $img_property
    fi
@@ -683,6 +694,8 @@ function setup_colorized_logging {
    iniset $conf_file DEFAULT logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_debug_format_suffix "[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s [01;35m%(instance)s[00m"
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_systemd_logging {
@@ -704,6 +717,9 @@ function setup_systemd_logging {
    iniset $conf_file DEFAULT logging_context_format_string "%(color)s%(levelname)s %(name)s [[01;36m%(global_request_id)s %(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_default_format_string "%(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "ERROR %(name)s [01;35m%(instance)s[00m"
+
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_standard_logging_identity {
@@ -49,7 +49,7 @@ KILL_PATH="$(which kill)"
 STACK_ENV_VARS="BASE_SQL_CONN DATA_DIR DEST ENABLED_SERVICES HOST_IP \
    KEYSTONE_SERVICE_URI \
    LOGFILE OS_CACERT SERVICE_HOST STACK_USER TLS_IP \
-    HOST_IPV6 SERVICE_IP_VERSION"
+    HOST_IPV6 SERVICE_IP_VERSION TUNNEL_ENDPOINT_IP TUNNEL_IP_VERSION"


 # Saves significant environment variables to .stackenv for later use
@@ -85,7 +85,7 @@ function write_clouds_yaml {
    if [ -f "$SSL_BUNDLE_FILE" ]; then
        CA_CERT_ARG="--os-cacert $SSL_BUNDLE_FILE"
    fi
-    # demo -> devstack
+    # devstack: user with the member role on demo project
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack \
@@ -96,18 +96,7 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-project-name demo

-    # alt_demo -> devstack-alt
-    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
-        --file $CLOUDS_YAML \
-        --os-cloud devstack-alt \
-        --os-region-name $REGION_NAME \
-        $CA_CERT_ARG \
-        --os-auth-url $KEYSTONE_SERVICE_URI \
-        --os-username alt_demo \
-        --os-password $ADMIN_PASSWORD \
-        --os-project-name alt_demo
-
-    # admin -> devstack-admin
+    # devstack-admin: user with the admin role on the admin project
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack-admin \
@@ -118,7 +107,62 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-project-name admin

-    # admin with a system-scoped token -> devstack-system
+    # devstack-admin-demo: user with the admin role on the demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-admin-demo \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username admin \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name demo
+
+    # devstack-alt: user with the member role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-alt-member: user with the member role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt-member \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo_member \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-alt-reader: user with the reader role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-reader: user with the reader role on demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username demo_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name demo
+
+    # devstack-system-admin: user with the admin role on the system
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack-system-admin \
@@ -129,6 +173,28 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-system-scope all

+    # devstack-system-member: user with the member role on the system
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-system-member \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username system_member \
+        --os-password $ADMIN_PASSWORD \
+        --os-system-scope all
+
+    # devstack-system-reader: user with the reader role on the system
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-system-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username system_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-system-scope all
+
    cat >> $CLOUDS_YAML <<EOF
 functional:
  image_name: $DEFAULT_IMAGE_NAME
@@ -170,6 +236,27 @@ function trueorfalse {
    $xtrace
 }

+# bool_to_int <True|False>
+#
+# Convert True|False to int 1 or 0
+# This function can be used to convert the output of trueorfalse
+# to an int follow c conventions where false is 0 and 1 it true.
+function bool_to_int {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    if [ -z $1 ]; then
+        die $LINENO "Bool value required"
+    fi
+    if [[ $1 == "True" ]] ; then
+        echo '1'
+    else
+        echo '0'
+    fi
+    $xtrace
+}
+
+
 function isset {
    [[ -v "$1" ]]
 }
@@ -314,9 +401,9 @@ function warn {
 # such as "install_package" further abstract things in better ways.
 #
 # ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
-# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
+# ``os_RELEASE`` - major release: ``22.04`` (Ubuntu), ``23`` (Fedora)
 # ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
-# ``os_CODENAME`` - vendor's codename for release: ``xenial``
+# ``os_CODENAME`` - vendor's codename for release: ``jammy``

 declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME

@@ -333,7 +420,7 @@ function _ensure_lsb_release {
    elif [[ -x $(command -v zypper 2>/dev/null) ]]; then
        sudo zypper -n install lsb-release
    elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
-        sudo dnf install -y redhat-lsb-core
+        sudo dnf install -y redhat-lsb-core || sudo dnf install -y openeuler-lsb
    else
        die $LINENO "Unable to find or auto-install lsb_release"
    fi
@@ -346,14 +433,24 @@ function _ensure_lsb_release {
 #  - os_VENDOR
 #  - os_PACKAGE
 function GetOSVersion {
-    # We only support distros that provide a sane lsb_release
-    _ensure_lsb_release
+    # CentOS Stream 9 and RHEL 9 do not provide lsb_release
+    source /etc/os-release
+    if [[ "${ID}${VERSION}" == "centos9" ]] || [[ "${ID}${VERSION}" =~ "rhel9" ]]; then
+        os_RELEASE=${VERSION_ID}
+        os_CODENAME="n/a"
+        os_VENDOR=$(echo $NAME | tr -d '[:space:]')
+    elif [[ "${ID}${VERSION}" =~ "rocky9" ]]; then
+        os_VENDOR="Rocky"
+        os_RELEASE=${VERSION_ID}
+    else
+        _ensure_lsb_release

-    os_RELEASE=$(lsb_release -r -s)
-    os_CODENAME=$(lsb_release -c -s)
-    os_VENDOR=$(lsb_release -i -s)
+        os_RELEASE=$(lsb_release -r -s)
+        os_CODENAME=$(lsb_release -c -s)
+        os_VENDOR=$(lsb_release -i -s)
+    fi

-    if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
+    if [[ $os_VENDOR =~ (Debian|Ubuntu) ]]; then
        os_PACKAGE="deb"
    else
        os_PACKAGE="rpm"
@@ -371,32 +468,25 @@ declare -g DISTRO

 function GetDistro {
    GetOSVersion
-    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
-            "$os_VENDOR" =~ (LinuxMint) ]]; then
-        # 'Everyone' refers to Ubuntu / Debian / Mint releases by
+    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
+        # 'Everyone' refers to Ubuntu / Debian releases by
        # the code name adjective
        DISTRO=$os_CODENAME
    elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
        # For Fedora, just use 'f' and the release
        DISTRO="f$os_RELEASE"
-    elif is_opensuse; then
-        DISTRO="opensuse-$os_RELEASE"
-        # Tumbleweed uses "n/a" as a codename, and the release is a datestring
-        # like 20180218, so not very useful. Leap however uses a release
-        # with a "dot", so for example 15.0
-        [ "$os_CODENAME" = "n/a" -a "$os_RELEASE" = "${os_RELEASE/\./}" ] && \
-            DISTRO="opensuse-tumbleweed"
-    elif is_suse_linux_enterprise; then
-        # just use major release
-        DISTRO="sle${os_RELEASE%.*}"
    elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
        "$os_VENDOR" =~ (CentOS) || \
+        "$os_VENDOR" =~ (AlmaLinux) || \
        "$os_VENDOR" =~ (Scientific) || \
        "$os_VENDOR" =~ (OracleServer) || \
+        "$os_VENDOR" =~ (Rocky) || \
        "$os_VENDOR" =~ (Virtuozzo) ]]; then
        # Drop the . release as we assume it's compatible
        # XXX re-evaluate when we get RHEL10
        DISTRO="rhel${os_RELEASE::1}"
+    elif [[ "$os_VENDOR" =~ (openEuler) ]]; then
+        DISTRO="openEuler-$os_RELEASE"
    else
        # We can't make a good choice here.  Setting a sensible DISTRO
        # is part of the problem, but not the major issue -- we really
@@ -440,7 +530,7 @@ function is_oraclelinux {


 # Determine if current distribution is a Fedora-based distribution
-# (Fedora, RHEL, CentOS, etc).
+# (Fedora, RHEL, CentOS, Rocky, etc).
 # is_fedora
 function is_fedora {
    if [[ -z "$os_VENDOR" ]]; then
@@ -448,44 +538,17 @@ function is_fedora {
    fi

    [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
+        [ "$os_VENDOR" = "openEuler" ] || \
        [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
        [ "$os_VENDOR" = "RedHatEnterprise" ] || \
+        [ "$os_VENDOR" = "RedHatEnterpriseLinux" ] || \
+        [ "$os_VENDOR" = "Rocky" ] || \
        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \
+        [ "$os_VENDOR" = "AlmaLinux" ] || \
        [ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ]
 }


-# Determine if current distribution is a SUSE-based distribution
-# (openSUSE, SLE).
-# is_suse
-function is_suse {
-    is_opensuse || is_suse_linux_enterprise
-}
-
-
-# Determine if current distribution is an openSUSE distribution
-# is_opensuse
-function is_opensuse {
-    if [[ -z "$os_VENDOR" ]]; then
-        GetOSVersion
-    fi
-
-    [[ "$os_VENDOR" =~ (openSUSE) ]]
-}
-
-
-# Determine if current distribution is a SUSE Linux Enterprise (SLE)
-# distribution
-# is_suse_linux_enterprise
-function is_suse_linux_enterprise {
-    if [[ -z "$os_VENDOR" ]]; then
-        GetOSVersion
-    fi
-
-    [[ "$os_VENDOR" =~ (^SUSE) ]]
-}
-
-
 # Determine if current distribution is an Ubuntu-based distribution
 # It will also detect non-Ubuntu but Debian-based distros
 # is_ubuntu
@@ -496,7 +559,14 @@ function is_ubuntu {
    [ "$os_PACKAGE" = "deb" ]
 }

-
+# Determine if current distribution is an openEuler distribution
+# is_openeuler
+function is_openeuler {
+    if [[ -z "$os_PACKAGE" ]]; then
+        GetOSVersion
+    fi
+    [ "$os_VENDOR" = "openEuler" ]
+}
 # Git Functions
 # =============

@@ -547,7 +617,7 @@ function git_clone {
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to 'required-projects' in the job definition."
-                die $LINENO "Cloning not allowed in this configuration"
+                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
            git_timed clone $git_clone_flags $git_remote $git_dest
        fi
@@ -559,10 +629,12 @@ function git_clone {
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to the \$PROJECTS variable in the job definition."
-                die $LINENO "Cloning not allowed in this configuration"
+                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
-            # '--branch' can also take tags
-            git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref
+            git_timed clone --no-checkout $git_clone_flags $git_remote $git_dest
+            cd $git_dest
+            git_timed fetch $git_clone_flags origin $git_ref
+            git_timed checkout FETCH_HEAD
        elif [[ "$RECLONE" = "True" ]]; then
            # if it does exist then simulate what clone does if asked to RECLONE
            cd $git_dest
@@ -572,7 +644,7 @@ function git_clone {
            # remove the existing ignored files (like pyc) as they cause breakage
            # (due to the py files having older timestamps than our pyc, so python
            # thinks the pyc files are correct using them)
-            find $git_dest -name '*.pyc' -delete
+            sudo find $git_dest -name '*.pyc' -delete

            # handle git_ref accordingly to type (tag, branch)
            if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
@@ -588,6 +660,18 @@ function git_clone {
        fi
    fi

+    # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
+    # about how we clone and work with repos.  Mark them safe globally
+    # as a work-around.
+    #
+    # NOTE(danms): On bionic (and likely others) git-config may write
+    # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+    # writes these changes to /etc/gitconfig which is more
+    # discoverable anyway.
+    #
+    # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
+    sudo git config --system --add safe.directory ${git_dest}
+
    # print out the results so we know what change was used in the logs
    cd $git_dest
    git show --oneline | head -1
@@ -687,7 +771,7 @@ function get_default_host_ip {
    if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
        host_ip=""
        # Find the interface used for the default route
-        host_ip_iface=${host_ip_iface:-$(ip -f $af route | awk '/default/ {print $5}' | head -1)}
+        host_ip_iface=${host_ip_iface:-$(ip -f $af route list match default table all | grep via | awk '/default/ {print $5}' | head -1)}
        local host_ips
        host_ips=$(LC_ALL=C ip -f $af addr show ${host_ip_iface} | sed /temporary/d |awk /$af'/ {split($2,parts,"/");  print parts[1]}')
        local ip
@@ -790,14 +874,9 @@ function policy_add {
 # Usage: get_or_create_domain <name> <description>
 function get_or_create_domain {
    local domain_id
-    # Gets domain id
    domain_id=$(
-        # Gets domain id
-        openstack domain show $1 \
-            -f value -c id 2>/dev/null ||
-        # Creates new domain
-        openstack domain create $1 \
-            --description "$2" \
+        openstack --os-cloud devstack-system-admin domain create $1 \
+            --description "$2" --or-show \
            -f value -c id
    )
    echo $domain_id
@@ -811,7 +890,7 @@ function get_or_create_group {
    # Gets group id
    group_id=$(
        # Creates new group with --or-show
-        openstack group create $1 \
+        openstack --os-cloud devstack-system-admin group create $1 \
            --domain $2 --description "$desc" --or-show \
            -f value -c id
    )
@@ -830,7 +909,7 @@ function get_or_create_user {
    # Gets user id
    user_id=$(
        # Creates new user with --or-show
-        openstack user create \
+        openstack --os-cloud devstack-system-admin user create \
            $1 \
            --password "$2" \
            --domain=$3 \
@@ -847,7 +926,7 @@ function get_or_create_project {
    local project_id
    project_id=$(
        # Creates new project with --or-show
-        openstack project create $1 \
+        openstack --os-cloud devstack-system-admin project create $1 \
            --domain=$2 \
            --or-show -f value -c id
    )
@@ -860,7 +939,7 @@ function get_or_create_role {
    local role_id
    role_id=$(
        # Creates role with --or-show
-        openstack role create $1 \
+        openstack --os-cloud devstack-system-admin role create $1 \
            --or-show -f value -c id
    )
    echo $role_id
@@ -886,29 +965,22 @@ function _get_domain_args {
 # Usage: get_or_add_user_project_role <role> <user> <project> [<user_domain> <project_domain>]
 function get_or_add_user_project_role {
    local user_role_id
+    local domain_args

    domain_args=$(_get_domain_args $4 $5)

-    # Gets user role id
-    user_role_id=$(openstack role assignment list \
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack --os-cloud devstack-system-admin role add $1 \
+        --user $2 \
+        --project $3 \
+        $domain_args
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --project $3 \
        $domain_args \
-        | grep '^|\s[a-f0-9]\+' | get_field 1)
-    if [[ -z "$user_role_id" ]]; then
-        # Adds role to user and get it
-        openstack role add $1 \
-            --user $2 \
-            --project $3 \
-            $domain_args
-        user_role_id=$(openstack role assignment list \
-            --role $1 \
-            --user $2 \
-            --project $3 \
-            $domain_args \
-            | grep '^|\s[a-f0-9]\+' | get_field 1)
-    fi
+        -c Role -f value)
    echo $user_role_id
 }

@@ -916,22 +988,48 @@ function get_or_add_user_project_role {
 # Usage: get_or_add_user_domain_role <role> <user> <domain>
 function get_or_add_user_domain_role {
    local user_role_id
-    # Gets user role id
-    user_role_id=$(openstack role assignment list \
+
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack --os-cloud devstack-system-admin role add $1 \
+        --user $2 \
+        --domain $3
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --domain $3 \
-        | grep '^|\s[a-f0-9]\+' | get_field 1)
+        -c Role -f value)
+
+    echo $user_role_id
+}
+
+# Gets or adds user role to system
+# Usage: get_or_add_user_system_role <role> <user> <system> [<user_domain>]
+function get_or_add_user_system_role {
+    local user_role_id
+    local domain_args
+
+    domain_args=$(_get_domain_args $4)
+
+    # Gets user role id
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
+        --role $1 \
+        --user $2 \
+        --system $3 \
+        $domain_args \
+        -f value -c Role)
    if [[ -z "$user_role_id" ]]; then
        # Adds role to user and get it
-        openstack role add $1 \
+        openstack --os-cloud devstack-system-admin role add $1 \
            --user $2 \
-            --domain $3
-        user_role_id=$(openstack role assignment list \
+            --system $3 \
+            $domain_args
+        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
            --role $1 \
            --user $2 \
-            --domain $3 \
-            | grep '^|\s[a-f0-9]\+' | get_field 1)
+            --system $3 \
+            $domain_args \
+            -f value -c Role)
    fi
    echo $user_role_id
 }
@@ -940,23 +1038,18 @@ function get_or_add_user_domain_role {
 # Usage: get_or_add_group_project_role <role> <group> <project>
 function get_or_add_group_project_role {
    local group_role_id
-    # Gets group role id
-    group_role_id=$(openstack role assignment list \
+
+    # Note this is idempotent so we are safe across multiple
+    # duplicate calls.
+    openstack role add $1 \
+        --group $2 \
+        --project $3
+    group_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --group $2 \
        --project $3 \
-        -f value)
-    if [[ -z "$group_role_id" ]]; then
-        # Adds role to group and get it
-        openstack role add $1 \
-            --group $2 \
-            --project $3
-        group_role_id=$(openstack role assignment list \
-            --role $1 \
-            --group $2 \
-            --project $3 \
-            -f value)
-    fi
+        -f value -c Role)
+
    echo $group_role_id
 }

@@ -967,9 +1060,9 @@ function get_or_create_service {
    # Gets service id
    service_id=$(
        # Gets service id
-        openstack service show $2 -f value -c id 2>/dev/null ||
+        openstack --os-cloud devstack-system-admin service show $2 -f value -c id 2>/dev/null ||
        # Creates new service if not exists
-        openstack service create \
+        openstack --os-cloud devstack-system-admin service create \
            $2 \
            --name $1 \
            --description="$3" \
@@ -982,14 +1075,14 @@ function get_or_create_service {
 # Usage: _get_or_create_endpoint_with_interface <service> <interface> <url> <region>
 function _get_or_create_endpoint_with_interface {
    local endpoint_id
-    endpoint_id=$(openstack endpoint list \
+    endpoint_id=$(openstack --os-cloud devstack-system-admin endpoint list \
        --service $1 \
        --interface $2 \
        --region $4 \
        -c ID -f value)
    if [[ -z "$endpoint_id" ]]; then
        # Creates new endpoint
-        endpoint_id=$(openstack endpoint create \
+        endpoint_id=$(openstack --os-cloud devstack-system-admin endpoint create \
            $1 $2 $3 --region $4 -f value -c id)
    fi

@@ -1023,7 +1116,7 @@ function get_or_create_endpoint {
 # Get a URL from the identity service
 # Usage: get_endpoint_url <service> <interface>
 function get_endpoint_url {
-    echo $(openstack endpoint list \
+    echo $(openstack --os-cloud devstack-system-admin endpoint list \
            --service $1 --interface $2 \
            -c URL -f value)
 }
@@ -1038,7 +1131,13 @@ function is_ironic_hardware {
 }

 function is_ironic_enforce_scope {
-    is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" ]] && return 0
+    is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]] && return 0
+    return 1
+}
+
+function is_ironic_sharded {
+    # todo(JayF): Support >1 shard with multiple n-cpu instances for each
+    is_service_enabled ironic && [[ "$IRONIC_SHARDS" == "1" ]] && return 0
    return 1
 }

@@ -1058,8 +1157,6 @@ function _get_package_dir {
        pkg_dir=$base_dir/debs
    elif is_fedora; then
        pkg_dir=$base_dir/rpms
-    elif is_suse; then
-        pkg_dir=$base_dir/rpms-suse
    else
        exit_distro_not_supported "list of packages"
    fi
@@ -1334,8 +1431,6 @@ function real_install_package {
        apt_get install "$@"
    elif is_fedora; then
        yum_install "$@"
-    elif is_suse; then
-        zypper_install "$@"
    else
        exit_distro_not_supported "installing packages"
    fi
@@ -1377,8 +1472,6 @@ function uninstall_package {
        apt_get purge "$@"
    elif is_fedora; then
        sudo dnf remove -y "$@" ||:
-    elif is_suse; then
-        sudo zypper remove -y "$@" ||:
    else
        exit_distro_not_supported "uninstalling packages"
    fi
@@ -1447,6 +1540,7 @@ function write_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local extra=""
    if [[ -n "$group" ]]; then
        extra="Group=$group"
@@ -1455,11 +1549,15 @@ function write_user_unit_file {
    mkdir -p $SYSTEMD_DIR

    iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
    iniset -sudo $unitfile "Service" "User" "$user"
    iniset -sudo $unitfile "Service" "ExecStart" "$command"
    iniset -sudo $unitfile "Service" "KillMode" "process"
    iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
    iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1474,10 +1572,12 @@ function write_uwsgi_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local unitfile="$SYSTEMD_DIR/$service"
    mkdir -p $SYSTEMD_DIR

    iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
+    iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
    iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
    iniset -sudo $unitfile "Service" "User" "$user"
    iniset -sudo $unitfile "Service" "ExecStart" "$command"
@@ -1488,6 +1588,9 @@ function write_uwsgi_user_unit_file {
    iniset -sudo $unitfile "Service" "NotifyAccess" "all"
    iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"

+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1535,10 +1638,17 @@ function _run_under_systemd {
    local systemd_service="devstack@$service.service"
    local group=$3
    local user=${4:-$STACK_USER}
+    if [[ -z "$user" ]]; then
+        user=$STACK_USER
+    fi
+    local env_vars="$5"
    if [[ "$command" =~ "uwsgi" ]] ; then
-        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        if [[ "$GLOBAL_VENV" == "True" ]] ; then
+            cmd="$cmd --venv $DEVSTACK_VENV"
+        fi
+        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    else
-        write_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    fi

    $SYSTEMCTL enable $systemd_service
@@ -1559,18 +1669,20 @@ function is_running {
 # If the command includes shell metachatacters (;<>*) it must be run using a shell
 # If an optional group is provided sg will be used to run the
 # command as that group.
-# run_process service "command-line" [group] [user]
+# run_process service "command-line" [group] [user] [env_vars]
+# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2"
 function run_process {
    local service=$1
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"

    local name=$service

    time_start "run_process"
    if is_service_enabled $service; then
-        _run_under_systemd "$name" "$command" "$group" "$user"
+        _run_under_systemd "$name" "$command" "$group" "$user" "$env_vars"
    fi
    time_stop "run_process"
 }
@@ -2326,6 +2438,11 @@ function time_stop {
    _TIME_TOTAL[$name]=$(($total + $elapsed_time))
 }

+function install_openstack_cli_server {
+    export PATH=$TOP_DIR/files/openstack-cli-server:$PATH
+    run_process openstack-cli-server "$PYTHON $TOP_DIR/files/openstack-cli-server/openstack-cli-server"
+}
+
 function oscwrap {
    local xtrace
    xtrace=$(set +o | grep xtrace)
@@ -2421,6 +2538,11 @@ function clean_pyc_files {
    fi
 }

+function is_fips_enabled {
+    fips=`cat /proc/sys/crypto/fips_enabled`
+    [ "$fips" == "1" ]
+}
+
 # Restore xtrace
 $_XTRACE_FUNCTIONS_COMMON

@@ -189,6 +189,9 @@ function iniset {
    local option=$3
    local value=$4

+    # Escape the ampersand character (&)
+    value=$(echo $value | sed -e 's/&/\\&/g')
+
    if [[ -z $section || -z $option ]]; then
        $xtrace
        return
@@ -7,7 +7,6 @@
 # External functions used:
 # - GetOSVersion
 # - is_fedora
-# - is_suse
 # - safe_chown

 # Save trace setting
@@ -33,6 +32,26 @@ function join_extras {
 # Python Functions
 # ================

+# Setup the global devstack virtualenvs and the associated environment
+# updates.
+function setup_devstack_virtualenv {
+    # We run devstack out of a global virtualenv.
+    if [[ ! -d $DEVSTACK_VENV ]] ; then
+        # Using system site packages to enable nova to use libguestfs.
+        # This package is currently installed via the distro and not
+        # available on pypi.
+        python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
+        pip_install -U pip setuptools
+        #NOTE(rpittau): workaround for simplejson removal in osc
+        #  https://review.opendev.org/c/openstack/python-openstackclient/+/920001
+        pip_install -U simplejson
+    fi
+    if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
+        export PATH="$DEVSTACK_VENV/bin:$PATH"
+        export PYTHON="$DEVSTACK_VENV/bin/python3"
+    fi
+}
+
 # Get the path to the pip command.
 # get_pip_command
 function get_pip_command {
@@ -61,9 +80,11 @@ function get_python_exec_prefix {
    fi
    $xtrace

-    local PYTHON_PATH=/usr/local/bin
-    is_suse && PYTHON_PATH=/usr/bin
-    echo $PYTHON_PATH
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        echo "$DEVSTACK_VENV/bin"
+    else
+        echo "/usr/local/bin"
+    fi
 }

 # Wrapper for ``pip install`` that only installs versions of libraries
@@ -168,6 +189,14 @@ function pip_install {
    if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
        local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
        local sudo_pip="env"
+    elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
+        # We have to check that the DEVSTACK_VENV exists because early
+        # devstack boostrapping needs to operate in a system context
+        # too bootstrap pip. Once pip is bootstrapped we create the
+        # global venv and can start to use it.
+        local cmd_pip=$DEVSTACK_VENV/bin/pip
+        local sudo_pip="env"
+        echo "Using python $PYTHON3_VERSION to install $package_dir"
    else
        local cmd_pip="python$PYTHON3_VERSION -m pip"
        # See
@@ -186,15 +215,11 @@ function pip_install {

    $xtrace

-    # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
-    # the same behaviour of setuptools before version 25.0.0.
-    # related issue: https://github.com/pypa/pip/issues/3874
    $sudo_pip \
        http_proxy="${http_proxy:-}" \
        https_proxy="${https_proxy:-}" \
        no_proxy="${no_proxy:-}" \
        PIP_FIND_LINKS=$PIP_FIND_LINKS \
-        SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
        $cmd_pip $upgrade \
        $@
    result=$?
@@ -383,6 +408,9 @@ function _setup_package_with_constraints_edit {
        # source we are about to do.
        local name
        name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
+        if [ -z $name ]; then
+            name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
+        fi
        $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
            $REQUIREMENTS_DIR/upper-constraints.txt -- $name
    fi
@@ -445,8 +473,11 @@ function setup_package {

    pip_install $flags "$project_dir$extras"
    # ensure that further actions can do things like setup.py sdist
-    if [[ "$flags" == "-e" ]]; then
-        safe_chown -R $STACK_USER $1/*.egg-info
+    if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
+        # egg-info is not created when project have pyproject.toml
+        if [ -d $1/*.egg-info ]; then
+            safe_chown -R $STACK_USER $1/*.egg-info
+        fi
    fi
 }

@@ -466,8 +497,6 @@ function install_python {
 function install_python3 {
    if is_ubuntu; then
        apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
-    elif is_suse; then
-        install_package python3-devel python3-dbm
    elif is_fedora; then
        if [ "$os_VENDOR" = "Fedora" ]; then
            install_package python${PYTHON3_VERSION//.}
@@ -60,6 +60,11 @@ function configure_rootwrap {
    sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf
    sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf

+    # Rely on $PATH set by devstack to determine what is safe to execute
+    # by rootwrap rather than use explicit whitelist of paths in
+    # rootwrap.conf
+    sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf
+
    # Set up the rootwrap sudoers
    local tempfile
    tempfile=$(mktemp)
@@ -27,6 +27,11 @@ set +o xtrace
 APACHE_USER=${APACHE_USER:-$STACK_USER}
 APACHE_GROUP=${APACHE_GROUP:-$(id -gn $APACHE_USER)}

+APACHE_LOCAL_HOST=$SERVICE_LOCAL_HOST
+if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
+    APACHE_LOCAL_HOST=[$APACHE_LOCAL_HOST]
+fi
+

 # Set up apache name and configuration directory
 # Note that APACHE_CONF_DIR is really more accurately apache's vhost
@@ -39,10 +44,6 @@ elif is_fedora; then
    APACHE_NAME=httpd
    APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/conf.d}
    APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
-elif is_suse; then
-    APACHE_NAME=apache2
-    APACHE_CONF_DIR=${APACHE_CONF_DIR:-/etc/$APACHE_NAME/vhosts.d}
-    APACHE_SETTINGS_DIR=${APACHE_SETTINGS_DIR:-/etc/$APACHE_NAME/conf.d}
 fi
 APACHE_LOG_DIR="/var/log/${APACHE_NAME}"

@@ -60,11 +61,6 @@ function enable_apache_mod {
            sudo a2enmod $mod
            restart_apache_server
        fi
-    elif is_suse; then
-        if ! a2enmod -q $mod ; then
-            sudo a2enmod $mod
-            restart_apache_server
-        fi
    elif is_fedora; then
        # pass
        true
@@ -82,19 +78,15 @@ function install_apache_uwsgi {
        apxs="apxs"
    fi

-    # This varies based on packaged/installed.  If we've
-    # pip_installed, then the pip setup will only build a "python"
-    # module that will be either python2 or python3 depending on what
-    # it was built with.
-    #
-    # For package installs, the distro ships both plugins and you need
-    # to select the right one ... it will not be autodetected.
-    UWSGI_PYTHON_PLUGIN=python3
-
    if is_ubuntu; then
        local pkg_list="uwsgi uwsgi-plugin-python3 libapache2-mod-proxy-uwsgi"
        install_package ${pkg_list}
-    elif is_fedora; then
+    # NOTE(ianw) 2022-02-03 : Fedora 35 needs to skip this and fall
+    # into the install-from-source because the upstream packages
+    # didn't fix Python 3.10 compatibility before release.  Should be
+    # fixed in uwsgi 4.9.0; can remove this when packages available
+    # or we drop this release
+    elif is_fedora && ! is_openeuler && ! [[ $DISTRO =~ f36 ]]; then
        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
        # default; the mod_proxy_uwsgi package actually conflicts now.
        # See:
@@ -103,10 +95,6 @@ function install_apache_uwsgi {
        # Thus there is nothing else to do after this install
        install_package uwsgi \
                        uwsgi-plugin-python3
-    elif [[ $os_VENDOR =~ openSUSE ]]; then
-        install_package uwsgi \
-                        uwsgi-python3 \
-                        apache2-mod_uwsgi
    else
        # Compile uwsgi from source.
        local dir
@@ -122,10 +110,9 @@ function install_apache_uwsgi {
        popd
        # delete the temp directory
        sudo rm -rf $dir
-        UWSGI_PYTHON_PLUGIN=python
    fi

-    if is_ubuntu || is_suse ; then
+    if is_ubuntu; then
        # we've got to enable proxy and proxy_uwsgi for this to work
        sudo a2enmod proxy
        sudo a2enmod proxy_uwsgi
@@ -150,13 +137,13 @@ function install_apache_wsgi {
    elif is_fedora; then
        sudo rm -f /etc/httpd/conf.d/000-*
        install_package httpd python3-mod_wsgi
+        # rpm distros dont enable httpd by default so enable it to support reboots.
+        sudo systemctl enable httpd
        # For consistency with Ubuntu, switch to the worker mpm, as
        # the default is event
        sudo sed -i '/mod_mpm_prefork.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
        sudo sed -i '/mod_mpm_event.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
        sudo sed -i '/mod_mpm_worker.so/s/^#//g' /etc/httpd/conf.modules.d/00-mpm.conf
-    elif is_suse; then
-        install_package apache2 apache2-mod_wsgi
    else
        exit_distro_not_supported "apache wsgi installation"
    fi
@@ -171,7 +158,7 @@ function install_apache_wsgi {
 # recognise it. a2ensite and a2dissite ignore the .conf suffix used as parameter. The default sites'
 # files are 000-default.conf and default-ssl.conf.
 #
-# On Fedora and openSUSE, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
+# On Fedora, any file in /etc/httpd/conf.d/ whose name ends with .conf is enabled.
 #
 # On RHEL and CentOS, things should hopefully work as in Fedora.
 #
@@ -187,7 +174,7 @@ function apache_site_config_for {
    if is_ubuntu; then
        # Ubuntu 14.04 - Apache 2.4
        echo $APACHE_CONF_DIR/${site}.conf
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        # fedora conf.d is only imported if it ends with .conf so this is approx the same
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        if [ -f $enabled_site_file ]; then
@@ -205,7 +192,7 @@ function enable_apache_site {
    enable_apache_mod version
    if is_ubuntu; then
        sudo a2ensite ${site}
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        # Do nothing if site already enabled or no site config exists
        if [[ -f ${enabled_site_file}.disabled ]] && [[ ! -f ${enabled_site_file} ]]; then
@@ -219,7 +206,7 @@ function disable_apache_site {
    local site=$@
    if is_ubuntu; then
        sudo a2dissite ${site} || true
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        local enabled_site_file="$APACHE_CONF_DIR/${site}.conf"
        # Do nothing if no site config exists
        if [[ -f ${enabled_site_file} ]]; then
@@ -250,13 +237,17 @@ function restart_apache_server {
    restart_service $APACHE_NAME
 }

+# write_uwsgi_config() - Create a new uWSGI config file
 function write_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
    local http=$4
-    local name=""
-    name=$(basename $wsgi)
+    local name=$5
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.
@@ -271,39 +262,47 @@ function write_uwsgi_config {
    local socket="$socket_dir/${name}.socket"

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
-    iniset "$file" uwsgi processes $API_WORKERS
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given. Note that the regex isn't exhaustive - neither Python modules
+    # nor Python variables can start with a number - but it's "good enough"
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
+    iniset "$conf" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
+    iniset "$conf" uwsgi lazy-apps true

    # If we said bind directly to http, then do that and don't start the apache proxy
    if [[ -n "$http" ]]; then
-        iniset "$file" uwsgi http $http
+        iniset "$conf" uwsgi http $http
    else
        local apache_conf=""
        apache_conf=$(apache_site_config_for $name)
-        iniset "$file" uwsgi socket "$socket"
-        iniset "$file" uwsgi chmod-socket 666
-        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee -a $apache_conf
+        iniset "$conf" uwsgi socket "$socket"
+        iniset "$conf" uwsgi chmod-socket 666
+        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 acquire=1 " | sudo tee -a $apache_conf
        enable_apache_site $name
        restart_apache_server
    fi
@@ -316,47 +315,58 @@ function write_uwsgi_config {
 # but that involves having apache buffer the request before sending it to
 # uwsgi.
 function write_local_uwsgi_http_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
-    name=$(basename $wsgi)
+    local name=$4
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
    port=$(get_random_port)
-    iniset "$file" uwsgi http-socket "127.0.0.1:$port"
-    iniset "$file" uwsgi processes $API_WORKERS
+    iniset "$conf" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN}
-    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
-    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
+    # uwsgi recommends this to prevent thundering herd on accept.
+    iniset "$conf" uwsgi thunder-lock true
+    # Set hook to trigger graceful shutdown on SIGTERM
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
-    iniset "$file" uwsgi chmod-socket 666
-    iniset "$file" uwsgi http-raw-body true
-    iniset "$file" uwsgi http-chunked-input true
-    iniset "$file" uwsgi http-auto-chunked true
-    iniset "$file" uwsgi http-keepalive false
+    iniset "$conf" uwsgi lazy-apps true
+    iniset "$conf" uwsgi chmod-socket 666
+    iniset "$conf" uwsgi http-raw-body true
+    iniset "$conf" uwsgi http-chunked-input true
+    iniset "$conf" uwsgi http-auto-chunked true
+    iniset "$conf" uwsgi http-keepalive false
    # Increase socket timeout for slow chunked uploads
-    iniset "$file" uwsgi socket-timeout 30
+    iniset "$conf" uwsgi socket-timeout 30

    enable_apache_mod proxy
    enable_apache_mod proxy_http
@@ -364,7 +374,7 @@ function write_local_uwsgi_http_config {
    apache_conf=$(apache_site_config_for $name)
    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${url}\" \"http://127.0.0.1:$port\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${url}\" \"http://$APACHE_LOCAL_HOST:$port\" retry=0 acquire=1 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }
@@ -383,18 +393,24 @@ function write_local_proxy_http_config {

    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${loc}\" \"$url\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${loc}\" \"$url\" retry=0 acquire=1 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }

 function remove_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local name=""
+    # TODO(stephenfin): Remove this call when everyone is using module path
+    # configuration instead of file path configuration
    name=$(basename $wsgi)

-    rm -rf $file
+    if [[ "$wsgi" = /* ]]; then
+        deprecated "Passing a wsgi script to remove_uwsgi_config is deprecated, pass an application name instead"
+    fi
+
+    rm -rf $conf
    disable_apache_site $name
 }

@@ -43,6 +43,13 @@ GITDIR["python-cinderclient"]=$DEST/python-cinderclient
 GITDIR["python-brick-cinderclient-ext"]=$DEST/python-brick-cinderclient-ext
 CINDER_DIR=$DEST/cinder

+if [[ $SERVICE_IP_VERSION == 6 ]]; then
+    CINDER_MY_IP="$HOST_IPV6"
+else
+    CINDER_MY_IP="$HOST_IP"
+fi
+
+
 # Cinder virtual environment
 if [[ ${USE_VENV} = True ]]; then
    PROJECT_VENV["cinder"]=${CINDER_DIR}.venv
@@ -69,6 +76,11 @@ CINDER_SERVICE_PORT_INT=${CINDER_SERVICE_PORT_INT:-18776}
 CINDER_SERVICE_PROTOCOL=${CINDER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 CINDER_SERVICE_LISTEN_ADDRESS=${CINDER_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)}

+# We do not need to report service status every 10s for devstack-like
+# deployments. In the gate this generates extra work for the services and the
+# database which are already taxed.
+CINDER_SERVICE_REPORT_INTERVAL=${CINDER_SERVICE_REPORT_INTERVAL:-120}
+
 # What type of LVM device should Cinder use for LVM backend
 # Defaults to auto, which will do thin provisioning if it's a fresh
 # volume group, otherwise it will do thick. The other valid choices are
@@ -76,6 +88,10 @@ CINDER_SERVICE_LISTEN_ADDRESS=${CINDER_SERVICE_LISTEN_ADDRESS:-$(ipv6_unquote $S
 # thin provisioning.
 CINDER_LVM_TYPE=${CINDER_LVM_TYPE:-auto}

+# ``CINDER_USE_SERVICE_TOKEN`` is a mode where service token is passed along with
+# user token while communicating to external REST APIs like Glance.
+CINDER_USE_SERVICE_TOKEN=$(trueorfalse True CINDER_USE_SERVICE_TOKEN)
+
 # Default backends
 # The backend format is type:name where type is one of the supported backend
 # types (lvm, nfs, etc) and name is the identifier used in the Cinder
@@ -88,13 +104,33 @@ CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
 CINDER_VOLUME_CLEAR=${CINDER_VOLUME_CLEAR:-${CINDER_VOLUME_CLEAR_DEFAULT:-zero}}
 CINDER_VOLUME_CLEAR=$(echo ${CINDER_VOLUME_CLEAR} | tr '[:upper:]' '[:lower:]')

-# Default to lioadm
-CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-lioadm}
+VOLUME_TYPE_MULTIATTACH=${VOLUME_TYPE_MULTIATTACH:-multiattach}

-# EL and SUSE should only use lioadm
-if is_fedora || is_suse; then
-    if [[ ${CINDER_ISCSI_HELPER} != "lioadm" ]]; then
-        die "lioadm is the only valid Cinder target_helper config on this platform"
+if [[ -n "$CINDER_ISCSI_HELPER" ]]; then
+    if [[ -z "$CINDER_TARGET_HELPER" ]]; then
+        deprecated 'Using CINDER_ISCSI_HELPER is deprecated, use CINDER_TARGET_HELPER instead'
+        CINDER_TARGET_HELPER="$CINDER_ISCSI_HELPER"
+    else
+        deprecated 'Deprecated CINDER_ISCSI_HELPER is set, but is being overwritten by CINDER_TARGET_HELPER'
+    fi
+fi
+CINDER_TARGET_HELPER=${CINDER_TARGET_HELPER:-lioadm}
+
+if [[ $CINDER_TARGET_HELPER == 'nvmet' ]]; then
+    CINDER_TARGET_PROTOCOL=${CINDER_TARGET_PROTOCOL:-'nvmet_rdma'}
+    CINDER_TARGET_PREFIX=${CINDER_TARGET_PREFIX:-'nvme-subsystem-1'}
+    CINDER_TARGET_PORT=${CINDER_TARGET_PORT:-4420}
+else
+    CINDER_TARGET_PROTOCOL=${CINDER_TARGET_PROTOCOL:-'iscsi'}
+    CINDER_TARGET_PREFIX=${CINDER_TARGET_PREFIX:-'iqn.2010-10.org.openstack:'}
+    CINDER_TARGET_PORT=${CINDER_TARGET_PORT:-3260}
+fi
+
+
+# EL should only use lioadm
+if is_fedora; then
+    if [[ ${CINDER_TARGET_HELPER} != "lioadm" && ${CINDER_TARGET_HELPER} != 'nvmet' ]]; then
+        die "lioadm and nvmet are the only valid Cinder target_helper config on this platform"
    fi
 fi

@@ -151,6 +187,12 @@ fi
 # Environment variables to configure the image-volume cache
 CINDER_IMG_CACHE_ENABLED=${CINDER_IMG_CACHE_ENABLED:-True}

+# Environment variables to configure the optimized volume upload
+CINDER_UPLOAD_OPTIMIZED=${CINDER_UPLOAD_OPTIMIZED:-False}
+
+# Environment variables to configure the internal tenant during optimized volume upload
+CINDER_UPLOAD_INTERNAL_TENANT=${CINDER_UPLOAD_INTERNAL_TENANT:-False}
+
 # For limits, if left unset, it will use cinder defaults of 0 for unlimited
 CINDER_IMG_CACHE_SIZE_GB=${CINDER_IMG_CACHE_SIZE_GB:-}
 CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
@@ -160,6 +202,11 @@ CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
 # enable the cache for all cinder backends.
 CINDER_CACHE_ENABLED_FOR_BACKENDS=${CINDER_CACHE_ENABLED_FOR_BACKENDS:-$CINDER_ENABLED_BACKENDS}

+# Configure which cinder backends will have optimized volume upload, this takes the same
+# form as the CINDER_ENABLED_BACKENDS config option. By default it will
+# enable the cache for all cinder backends.
+CINDER_UPLOAD_OPTIMIZED_BACKENDS=${CINDER_UPLOAD_OPTIMIZED_BACKENDS:-$CINDER_ENABLED_BACKENDS}
+
 # Flag to set the oslo_policy.enforce_scope. This is used to switch
 # the  Volume API policies to start checking the scope of token. by default,
 # this flag is False.
@@ -187,7 +234,7 @@ function _cinder_cleanup_apache_wsgi {
 function cleanup_cinder {
    # ensure the volume group is cleared up because fails might
    # leave dead volumes in the group
-    if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
+    if [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
        local targets
        targets=$(sudo tgtadm --op show --mode target)
        if [ $? -ne 0 ]; then
@@ -215,8 +262,14 @@ function cleanup_cinder {
        else
            stop_service tgtd
        fi
-    else
+    elif [ "$CINDER_TARGET_HELPER" = "lioadm" ]; then
        sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
+    elif [ "$CINDER_TARGET_HELPER" = "nvmet" ]; then
+        # If we don't disconnect everything vgremove will block
+        sudo nvme disconnect-all
+        sudo nvmetcli clear
+    else
+        die $LINENO "Unknown value \"$CINDER_TARGET_HELPER\" for CINDER_TARGET_HELPER"
    fi

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
@@ -237,7 +290,7 @@ function cleanup_cinder {
    fi

    stop_process "c-api"
-    remove_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI"
+    remove_uwsgi_config "$CINDER_UWSGI_CONF" "cinder-wsgi"
 }

 # configure_cinder() - Set config files, create data dirs, etc
@@ -267,7 +320,7 @@ function configure_cinder {

    iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

-    iniset $CINDER_CONF DEFAULT target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF DEFAULT target_helper "$CINDER_TARGET_HELPER"
    iniset $CINDER_CONF database connection `database_connection_url cinder`
    iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
    iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"
@@ -275,22 +328,26 @@ function configure_cinder {
    iniset $CINDER_CONF DEFAULT osapi_volume_listen $CINDER_SERVICE_LISTEN_ADDRESS
    iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH
    iniset $CINDER_CONF oslo_concurrency lock_path $CINDER_STATE_PATH
-    if [[ $SERVICE_IP_VERSION == 6 ]]; then
-        iniset $CINDER_CONF DEFAULT my_ip "$HOST_IPV6"
-    else
-        iniset $CINDER_CONF DEFAULT my_ip "$HOST_IP"
-    fi
+    iniset $CINDER_CONF DEFAULT my_ip "$CINDER_MY_IP"
    iniset $CINDER_CONF key_manager backend cinder.keymgr.conf_key_mgr.ConfKeyManager
    iniset $CINDER_CONF key_manager fixed_key $(openssl rand -hex 16)
    if [[ -n "$CINDER_ALLOWED_DIRECT_URL_SCHEMES" ]]; then
        iniset $CINDER_CONF DEFAULT allowed_direct_url_schemes $CINDER_ALLOWED_DIRECT_URL_SCHEMES
    fi

+    # set default quotas
+    iniset $CINDER_CONF DEFAULT quota_volumes ${CINDER_QUOTA_VOLUMES:-10}
+    iniset $CINDER_CONF DEFAULT quota_backups ${CINDER_QUOTA_BACKUPS:-10}
+    iniset $CINDER_CONF DEFAULT quota_snapshots ${CINDER_QUOTA_SNAPSHOTS:-10}
+
    # Avoid RPC timeouts in slow CI and test environments by doubling the
    # default response timeout set by RPC clients. See bug #1873234 for more
    # details and example failures.
    iniset $CINDER_CONF DEFAULT rpc_response_timeout 120

+    iniset $CINDER_CONF DEFAULT report_interval $CINDER_SERVICE_REPORT_INTERVAL
+    iniset $CINDER_CONF DEFAULT service_down_time $(($CINDER_SERVICE_REPORT_INTERVAL * 6))
+
    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local enabled_backends=""
        local default_name=""
@@ -311,6 +368,14 @@ function configure_cinder {
            iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}
        fi
        configure_cinder_image_volume_cache
+
+        # The upload optimization uses Cinder's clone volume functionality to
+        # clone the Image-Volume from source volume hence can only be
+        # performed when glance is using cinder as it's backend.
+        if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then
+            # Configure optimized volume upload
+            configure_cinder_volume_upload
+        fi
    fi

    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
@@ -348,7 +413,9 @@ function configure_cinder {
    # Format logging
    setup_logging $CINDER_CONF $CINDER_USE_MOD_WSGI

-    write_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI" "/volume"
+    if is_service_enabled c-api; then
+        write_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI" "/volume"
+    fi

    if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
        configure_cinder_driver
@@ -370,54 +437,58 @@ function configure_cinder {
    if [[ ! -z "$CINDER_COORDINATION_URL" ]]; then
        iniset $CINDER_CONF coordination backend_url "$CINDER_COORDINATION_URL"
    elif is_service_enabled etcd3; then
-        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT"
+        # NOTE(jan.gutter): api_version can revert to default once tooz is
+        # updated with the etcd v3.4 defaults
+        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT?api_version=v3"
    fi

-    if [[ "$CINDER_ENFORCE_SCOPE" == True ]] ; then
+    if [[ "$CINDER_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
        iniset $CINDER_CONF oslo_policy enforce_scope true
        iniset $CINDER_CONF oslo_policy enforce_new_defaults true
+    else
+        iniset $CINDER_CONF oslo_policy enforce_scope false
+        iniset $CINDER_CONF oslo_policy enforce_new_defaults false
+    fi
+
+    if [ "$CINDER_USE_SERVICE_TOKEN" == "True" ]; then
+        init_cinder_service_user_conf
    fi
 }

 # create_cinder_accounts() - Set up common required cinder accounts

-# Tenant               User       Roles
+# Project              User       Roles
 # ------------------------------------------------------------------
-# service              cinder     admin        # if enabled
+# SERVICE_PROJECT_NAME cinder     service
+# SERVICE_PROJECT_NAME cinder     creator (if Barbican is enabled)

 # Migrated from keystone_data.sh
 function create_cinder_accounts {
    # Cinder
    if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

-        create_service_user "cinder"
+        local extra_role=""
+
+        # cinder needs the "creator" role in order to interact with barbican
+        if is_service_enabled barbican; then
+            extra_role=$(get_or_create_role "creator")
+        fi
+
+        create_service_user "cinder" $extra_role
+
+        local cinder_api_url
+        if [[ "$CINDER_USE_MOD_WSGI" == "False" ]]; then
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT"
+        else
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume"
+        fi

        # block-storage is the official service type
        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
-        if [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-        else
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-        fi
-
+        get_or_create_endpoint \
+            "block-storage" \
+            "$REGION_NAME" \
+            "$cinder_api_url/v3"
        configure_cinder_internal_tenant
    fi
 }
@@ -440,10 +511,6 @@ function init_cinder {
            be_type=${be%%:*}
            be_name=${be##*:}
            if type init_cinder_backend_${be_type} >/dev/null 2>&1; then
-                # Always init the default volume group for lvm.
-                if [[ "$be_type" == "lvm" ]]; then
-                    init_default_lvm_volume_group
-                fi
                init_cinder_backend_${be_type} ${be_name}
            fi
        done
@@ -462,9 +529,9 @@ function init_cinder {
 function install_cinder {
    git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH
    setup_develop $CINDER_DIR
-    if [[ "$CINDER_ISCSI_HELPER" == "tgtadm" ]]; then
+    if [[ "$CINDER_TARGET_HELPER" == "tgtadm" ]]; then
        install_package tgt
-    elif [[ "$CINDER_ISCSI_HELPER" == "lioadm" ]]; then
+    elif [[ "$CINDER_TARGET_HELPER" == "lioadm" ]]; then
        if is_ubuntu; then
            # TODO(frickler): Workaround for https://launchpad.net/bugs/1819819
            sudo mkdir -p /etc/target
@@ -473,6 +540,43 @@ function install_cinder {
        else
            install_package targetcli
        fi
+    elif [[ "$CINDER_TARGET_HELPER" == "nvmet" ]]; then
+        install_package nvme-cli
+
+        # TODO: Remove manual installation of the dependency when the
+        # requirement is added to nvmetcli:
+        # http://lists.infradead.org/pipermail/linux-nvme/2022-July/033576.html
+        if is_ubuntu; then
+            install_package python3-configshell-fb
+        else
+            install_package python3-configshell
+        fi
+        # Install from source because Ubuntu doesn't have the package and some packaged versions didn't work on Python 3
+        pip_install git+git://git.infradead.org/users/hch/nvmetcli.git
+
+        sudo modprobe nvmet
+        sudo modprobe nvme-fabrics
+
+        if [[ $CINDER_TARGET_PROTOCOL == 'nvmet_rdma' ]]; then
+            install_package rdma-core
+            sudo modprobe nvme-rdma
+
+            # Create the Soft-RoCE device over the networking interface
+            local iface=${HOST_IP_IFACE:-`ip -br -$SERVICE_IP_VERSION a | grep $CINDER_MY_IP | awk '{print $1}'`}
+            if [[ -z "$iface" ]]; then
+                die $LINENO "Cannot find interface to bind Soft-RoCE"
+            fi
+
+            if ! sudo rdma link | grep $iface ; then
+                sudo rdma link add rxe_$iface type rxe netdev $iface
+            fi
+
+        elif [[ $CINDER_TARGET_PROTOCOL == 'nvmet_tcp' ]]; then
+            sudo modprobe nvme-tcp
+
+        else  # 'nvmet_fc'
+            sudo modprobe nvme-fc
+        fi
    fi
 }

@@ -509,18 +613,13 @@ function start_cinder {
        service_port=$CINDER_SERVICE_PORT_INT
        service_protocol="http"
    fi
-    if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
+    if [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
        if is_service_enabled c-vol; then
            # Delete any old stack.conf
            sudo rm -f /etc/tgt/conf.d/stack.conf
            _configure_tgt_for_config_d
            if is_ubuntu; then
                sudo service tgt restart
-            elif is_suse; then
-                # NOTE(dmllr): workaround restart bug
-                # https://bugzilla.suse.com/show_bug.cgi?id=934642
-                stop_service tgtd
-                start_service tgtd
            else
                restart_service tgtd
            fi
@@ -549,8 +648,13 @@ function start_cinder {
    fi

    run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
-    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
-    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
+    # Tune glibc for Python Services using single malloc arena for all threads
+    # and disabling dynamic thresholds to reduce memory usage when using native
+    # threads directly or via eventlet.tpool
+    # https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html
+    malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144"
+    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning"
+    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning"

    # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
    # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
@@ -565,6 +669,23 @@ function stop_cinder {
    stop_process c-vol
 }

+function create_one_type {
+    type_name=$1
+    property_key=$2
+    property_value=$3
+    # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
+    if is_service_enabled keystone; then
+        openstack --os-region-name="$REGION_NAME" volume type create --property $property_key="$property_value" $type_name
+    else
+        # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
+        # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
+        local cinder_url
+        cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create $type_name
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key $type_name set $property_key="$property_value"
+    fi
+}
+
 # create_volume_types() - Create Cinder's configured volume types
 function create_volume_types {
    # Create volume types
@@ -572,19 +693,13 @@ function create_volume_types {
        local be be_name
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_name=${be##*:}
-            # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
-            if is_service_enabled keystone; then
-                openstack --os-region-name="$REGION_NAME" volume type create --property volume_backend_name="${be_name}" ${be_name}
-            else
-                # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
-                # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
-                local cinder_url
-                cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create ${be_name}
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key ${be_name} set volume_backend_name=${be_name}
-            fi
+            create_one_type $be_name "volume_backend_name" $be_name
        done

+        if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then
+            create_one_type $VOLUME_TYPE_MULTIATTACH $VOLUME_TYPE_MULTIATTACH "<is> True"
+        fi
+
        # Increase quota for the service project if glance is using cinder,
        # since it's likely to occasionally go above the default 10 in parallel
        # test execution.
@@ -628,6 +743,24 @@ function configure_cinder_image_volume_cache {
    done
 }

+function configure_cinder_volume_upload {
+    # Expect UPLOAD_VOLUME_OPTIMIZED_FOR_BACKENDS to be a list of backends
+    # similar to CINDER_ENABLED_BACKENDS with NAME:TYPE where NAME will
+    # be the backend specific configuration stanza in cinder.conf.
+    local be be_name
+    for be in ${CINDER_UPLOAD_OPTIMIZED_BACKENDS//,/ }; do
+        be_name=${be##*:}
+
+        iniset $CINDER_CONF $be_name image_upload_use_cinder_backend $CINDER_UPLOAD_OPTIMIZED
+        iniset $CINDER_CONF $be_name image_upload_use_internal_tenant $CINDER_UPLOAD_INTERNAL_TENANT
+    done
+}
+
+function init_cinder_service_user_conf {
+    configure_keystone_authtoken_middleware $CINDER_CONF cinder service_user
+    iniset $CINDER_CONF service_user send_service_user_token True
+    iniset $CINDER_CONF service_user auth_strategy keystone
+}

 # Restore xtrace
 $_XTRACE_CINDER
@@ -50,7 +50,7 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.tests.fake_driver.FakeGateDriver"
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
-    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF $be_name target_helper "$CINDER_TARGET_HELPER"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"

    if [[ "$CINDER_VOLUME_CLEAR" == "non" ]]; then
@@ -50,7 +50,10 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMVolumeDriver"
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
-    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
+    iniset $CINDER_CONF $be_name target_helper "$CINDER_TARGET_HELPER"
+    iniset $CINDER_CONF $be_name target_protocol "$CINDER_TARGET_PROTOCOL"
+    iniset $CINDER_CONF $be_name target_port "$CINDER_TARGET_PORT"
+    iniset $CINDER_CONF $be_name target_prefix "$CINDER_TARGET_PREFIX"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"
    iniset $CINDER_CONF $be_name volume_clear "$CINDER_VOLUME_CLEAR"
 }
@@ -32,6 +32,15 @@ function configure_cinder_backend_nfs {
    iniset $CINDER_CONF $be_name volume_backend_name $be_name
    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.nfs.NfsDriver"
    iniset $CINDER_CONF $be_name nfs_shares_config "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
+    iniset $CINDER_CONF $be_name nas_host localhost
+    iniset $CINDER_CONF $be_name nas_share_path ${NFS_EXPORT_DIR}
+    iniset $CINDER_CONF $be_name nas_secure_file_operations \
+        ${NFS_SECURE_FILE_OPERATIONS}
+    iniset $CINDER_CONF $be_name nas_secure_file_permissions \
+        ${NFS_SECURE_FILE_PERMISSIONS}
+
+    # NFS snapshot support is currently opt-in only.
+    iniset $CINDER_CONF $be_name nfs_snapshot_support True

    echo "$CINDER_NFS_SERVERPATH" | tee "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
 }
@@ -26,12 +26,15 @@ CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}


 function configure_cinder_backup_ceph {
-    sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
-    if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
-        sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+    # Execute this part only when cephadm is not used
+    if [[ "$CEPHADM_DEPLOY" = "False" ]]; then
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
+        if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
+            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+        fi
+        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
    fi
-    sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-    sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring

    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
    iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
@@ -24,6 +24,9 @@ function configure_cinder_backup_swift {
    # to use it.
    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.swift.SwiftBackupDriver"
    iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    if is_service_enabled tls-proxy; then
+        iniset $CINDER_CONF DEFAULT backup_swift_ca_cert_file $SSL_BUNDLE_FILE
+    fi
 }

 # init_cinder_backup_swift: nothing to do
@@ -89,6 +89,10 @@ function initialize_database_backends {
        DATABASE_PASSWORD=$MYSQL_PASSWORD
    fi

+    return 0
+}
+
+function define_database_baseurl {
    # We configure Nova, Horizon, Glance and Keystone to use MySQL as their
    # database server.  While they share a single server, each has their own
    # database and tables.
@@ -100,8 +104,6 @@ function initialize_database_backends {

    # NOTE: Don't specify ``/db`` in this string so we can use it for multiple services
    BASE_SQL_CONN=${BASE_SQL_CONN:-$(get_database_type_$DATABASE_TYPE)://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST}
-
-    return 0
 }

 # Recreate a given database
@@ -12,6 +12,7 @@ _XTRACE_DB_MYSQL=$(set +o | grep xtrace)
 set +o xtrace

 MYSQL_DRIVER=${MYSQL_DRIVER:-PyMySQL}
+INSTALL_DATABASE_SERVER_PACKAGES=$(trueorfalse True INSTALL_DATABASE_SERVER_PACKAGES)

 register_database mysql

@@ -19,11 +20,7 @@ if [[ -z "$MYSQL_SERVICE_NAME" ]]; then
    MYSQL_SERVICE_NAME=mysql
    if is_fedora && ! is_oraclelinux; then
        MYSQL_SERVICE_NAME=mariadb
-    elif is_suse && systemctl list-unit-files | grep -q 'mariadb\.service'; then
-        # Older mariadb packages on SLES 12 provided mysql.service.  The
-        # newer ones on SLES 12 and 15 use mariadb.service; they also
-        # provide a mysql.service symlink for backwards-compatibility, but
-        # let's not rely on that.
+    elif [[ "$DISTRO" =~ bookworm|bullseye ]]; then
        MYSQL_SERVICE_NAME=mariadb
    fi
 fi
@@ -51,7 +48,7 @@ function cleanup_database_mysql {
    elif is_oraclelinux; then
        uninstall_package mysql-community-server
        sudo rm -rf /var/lib/mysql
-    elif is_suse || is_fedora; then
+    elif is_fedora; then
        uninstall_package mariadb-server
        sudo rm -rf /var/lib/mysql
    else
@@ -66,12 +63,12 @@ function recreate_database_mysql {
 }

 function configure_database_mysql {
-    local my_conf mysql slow_log
+    local my_conf mysql slow_log my_client_conf
    echo_summary "Configuring and starting MySQL"

    if is_ubuntu; then
        my_conf=/etc/mysql/my.cnf
-    elif is_suse || is_oraclelinux; then
+    elif is_oraclelinux; then
        my_conf=/etc/my.cnf
    elif is_fedora; then
        my_conf=/etc/my.cnf
@@ -83,42 +80,69 @@ function configure_database_mysql {
        exit_distro_not_supported "mysql configuration"
    fi

-    # Start mysql-server
-    if is_fedora || is_suse; then
+    # Set fips mode on
+    if is_ubuntu; then
+        if is_fips_enabled; then
+            my_client_conf=/etc/mysql/mysql.conf.d/mysql.cnf
+            iniset -sudo $my_client_conf mysql ssl-fips-mode "on"
+            iniset -sudo $my_conf mysqld ssl-fips-mode "on"
+        fi
+    fi
+
+    # Change bind-address from localhost (127.0.0.1) to any (::)
+    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"
+
+    # (Re)Start mysql-server
+    if is_fedora; then
        # service is not started by default
        start_service $MYSQL_SERVICE_NAME
+    elif is_ubuntu; then
+        # required since bind-address could have changed above
+        restart_service $MYSQL_SERVICE_NAME
    fi

    # Set the root password - only works the first time. For Ubuntu, we already
    # did that with debconf before installing the package, but we still try,
-    # because the package might have been installed already.
-    sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+    # because the package might have been installed already. We don't do this
+    # for Ubuntu 22.04 (jammy) because the authorization model change in
+    # version 10.4 of mariadb. See
+    # https://mariadb.org/authentication-in-mariadb-10-4/
+    if ! (is_ubuntu && [[ "$DISTRO" == "jammy" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then
+        sudo mysqladmin -u root password $DATABASE_PASSWORD || true
+    fi

    # In case of Mariadb, giving hostname in arguments causes permission
    # problems as it expects connection through socket
    if is_ubuntu && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
        local cmd_args="-uroot -p$DATABASE_PASSWORD "
    else
-        local cmd_args="-uroot -p$DATABASE_PASSWORD -h127.0.0.1 "
+        local cmd_args="-uroot -p$DATABASE_PASSWORD -h$SERVICE_LOCAL_HOST "
    fi

    # In mariadb e.g. on Ubuntu socket plugin is used for authentication
    # as root so it works only as sudo. To restore old "mysql like" behaviour,
    # we need to change auth plugin for root user
-    if is_ubuntu && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
-        sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
-        sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+    # TODO(frickler): simplify this logic
+    if is_ubuntu && [[ ! "$DISTRO" =~ bookworm|bullseye ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
+        if [[ "$DISTRO" == "jammy" ]]; then
+            # For Ubuntu 22.04 (jammy) we follow the model outlined in
+            # https://mariadb.org/authentication-in-mariadb-10-4/
+            sudo mysql -e "ALTER USER $DATABASE_USER@localhost IDENTIFIED VIA mysql_native_password USING PASSWORD('$DATABASE_PASSWORD');"
+        else
+            sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
+            sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+        fi
+    fi
+    if ! (is_ubuntu && [[ "$DISTRO" == "jammy" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]); then
+        # Create DB user if it does not already exist
+        sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
+        # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
+        sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"
    fi
-    # Create DB user if it does not already exist
-    sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
-    # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
-    sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"

    # Now update ``my.cnf`` for some local needs and restart the mysql service

-    # Change bind-address from localhost (127.0.0.1) to any (::) and
-    # set default db type to InnoDB
-    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"
+    # Set default db type to InnoDB
    iniset -sudo $my_conf mysqld sql_mode TRADITIONAL
    iniset -sudo $my_conf mysqld default-storage-engine InnoDB
    iniset -sudo $my_conf mysqld max_connections 1024
@@ -143,6 +167,29 @@ function configure_database_mysql {
        iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1
    fi

+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        echo "enabling MySQL performance counting"
+
+        # Install our sqlalchemy plugin
+        pip_install ${TOP_DIR}/tools/dbcounter
+
+        # Create our stats database for accounting
+        recreate_database stats
+        mysql -u $DATABASE_USER -p$DATABASE_PASSWORD -h $MYSQL_HOST -e \
+              "CREATE TABLE queries (db VARCHAR(32), op VARCHAR(32),
+                count INT, PRIMARY KEY (db, op)) ENGINE MEMORY" stats
+    fi
+
+    if [[ "$MYSQL_REDUCE_MEMORY" == "True" ]]; then
+        iniset -sudo $my_conf mysqld read_buffer_size 64K
+        iniset -sudo $my_conf mysqld innodb_buffer_pool_size 16M
+        iniset -sudo $my_conf mysqld thread_stack 192K
+        iniset -sudo $my_conf mysqld thread_cache_size 8
+        iniset -sudo $my_conf mysqld tmp_table_size 8M
+        iniset -sudo $my_conf mysqld sort_buffer_size 8M
+        iniset -sudo $my_conf mysqld max_allowed_packet 8M
+    fi
+
    restart_service $MYSQL_SERVICE_NAME
 }

@@ -173,18 +220,17 @@ EOF
        chmod 0600 $HOME/.my.cnf
    fi
    # Install mysql-server
-    if is_oraclelinux; then
-        install_package mysql-community-server
-    elif is_fedora; then
-        install_package mariadb-server mariadb-devel
-        sudo systemctl enable $MYSQL_SERVICE_NAME
-    elif is_suse; then
-        install_package mariadb-server
-        sudo systemctl enable $MYSQL_SERVICE_NAME
-    elif is_ubuntu; then
-        install_package $MYSQL_SERVICE_NAME-server
-    else
-        exit_distro_not_supported "mysql installation"
+    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
+        if is_oraclelinux; then
+            install_package mysql-community-server
+        elif is_fedora; then
+            install_package mariadb-server mariadb-devel mariadb
+            sudo systemctl enable $MYSQL_SERVICE_NAME
+        elif is_ubuntu; then
+            install_package $MYSQL_SERVICE_NAME-server
+        else
+            exit_distro_not_supported "mysql installation"
+        fi
    fi
 }

@@ -200,7 +246,17 @@ function install_database_python_mysql {

 function database_connection_url_mysql {
    local db=$1
-    echo "$BASE_SQL_CONN/$db?charset=utf8"
+    local plugin
+
+    # NOTE(danms): We don't enable perf on subnodes yet because the
+    # plugin is not installed there
+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        if is_service_enabled mysql; then
+            plugin="&plugin=dbcounter"
+        fi
+    fi
+
+    echo "$BASE_SQL_CONN/$db?charset=utf8$plugin"
 }


@@ -13,7 +13,7 @@ set +o xtrace


 MAX_DB_CONNECTIONS=${MAX_DB_CONNECTIONS:-200}
-
+INSTALL_DATABASE_SERVER_PACKAGES=$(trueorfalse True INSTALL_DATABASE_SERVER_PACKAGES)

 register_database postgresql

@@ -32,7 +32,7 @@ function cleanup_database_postgresql {
        # Get ruthless with mysql
        apt_get purge -y postgresql*
        return
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        uninstall_package postgresql-server
    else
        return
@@ -46,6 +46,10 @@ function recreate_database_postgresql {
    createdb -h $DATABASE_HOST -U$DATABASE_USER -l C -T template0 -E utf8 $db
 }

+function _exit_pg_init {
+    sudo cat /var/lib/pgsql/initdb_postgresql.log
+}
+
 function configure_database_postgresql {
    local pg_conf pg_dir pg_hba check_role version
    echo_summary "Configuring and starting PostgreSQL"
@@ -53,7 +57,9 @@ function configure_database_postgresql {
        pg_hba=/var/lib/pgsql/data/pg_hba.conf
        pg_conf=/var/lib/pgsql/data/postgresql.conf
        if ! sudo [ -e $pg_hba ]; then
+            trap _exit_pg_init EXIT
            sudo postgresql-setup initdb
+            trap - EXIT
        fi
    elif is_ubuntu; then
        version=`psql --version | cut -d ' ' -f3 | cut -d. -f1-2`
@@ -66,11 +72,6 @@ function configure_database_postgresql {
        pg_dir=`find /etc/postgresql -name pg_hba.conf|xargs dirname`
        pg_hba=$pg_dir/pg_hba.conf
        pg_conf=$pg_dir/postgresql.conf
-    elif is_suse; then
-        pg_hba=/var/lib/pgsql/data/pg_hba.conf
-        pg_conf=/var/lib/pgsql/data/postgresql.conf
-        # initdb is called when postgresql is first started
-        sudo [ -e $pg_hba ] || start_service postgresql
    else
        exit_distro_not_supported "postgresql configuration"
    fi
@@ -95,7 +96,6 @@ function configure_database_postgresql {

 function install_database_postgresql {
    echo_summary "Installing postgresql"
-    deprecated "Use of postgresql in devstack is deprecated, and will be removed during the Pike cycle"
    local pgpass=$HOME/.pgpass
    if [[ ! -e $pgpass ]]; then
        cat <<EOF > $pgpass
@@ -105,15 +105,17 @@ EOF
    else
        sed -i "s/:root:\w\+/:root:$DATABASE_PASSWORD/" $pgpass
    fi
-    if is_ubuntu; then
-        install_package postgresql
-    elif is_fedora || is_suse; then
-        install_package postgresql-server
-        if is_fedora; then
-            sudo systemctl enable postgresql
+    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
+        if is_ubuntu; then
+            install_package postgresql
+        elif is_fedora; then
+            install_package postgresql-server
+            if is_fedora; then
+                sudo systemctl enable postgresql
+            fi
+        else
+            exit_distro_not_supported "postgresql installation"
        fi
-    else
-        exit_distro_not_supported "postgresql installation"
    fi
 }

@@ -40,12 +40,18 @@ function start_dstat {
    if is_service_enabled peakmem_tracker; then
        die $LINENO "The peakmem_tracker service has been removed, use memory_tracker instead"
    fi
+
+    # To enable file_tracker add:
+    #    enable_service file_tracker
+    # to your localrc
+    run_process file_tracker "$TOP_DIR/tools/file_tracker.sh"
 }

 # stop_dstat() stop dstat process
 function stop_dstat {
    stop_process dstat
    stop_process memory_tracker
+    stop_process file_tracker
 }

 # Restore xtrace
@@ -51,7 +51,7 @@ function start_etcd3 {
    fi
    cmd+=" --listen-client-urls http://$SERVICE_HOST:$ETCD_PORT"
    if [ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]; then
-        cmd+=" --debug"
+        cmd+=" --log-level=debug"
    fi

    local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
@@ -47,9 +47,8 @@ USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE)
 # from CINDER_ENABLED_BACKENDS
 GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
 GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
-# NOTE (abhishekk): For opensuse data files are stored in different directory
-if is_opensuse; then
-    GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/etc/glance
+if [[ "$GLOBAL_VENV" == "True" ]] ; then
+    GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance
 fi
 # When Cinder is used as a glance store, you can optionally configure cinder to
 # optimize bootable volume creation by allowing volumes to be cloned directly
@@ -76,6 +75,7 @@ GLANCE_MULTIPLE_FILE_STORES=${GLANCE_MULTIPLE_FILE_STORES:-fast}
 GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:-fast}

 GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache}
+GLANCE_CACHE_DRIVER=${GLANCE_CACHE_DRIVER:-centralized_db}

 # Full Glance functionality requires running in standalone mode. If we are
 # not in uwsgi mode, then we are standalone, otherwise allow separate control.
@@ -99,10 +99,13 @@ GLANCE_USE_IMPORT_WORKFLOW=$(trueorfalse False GLANCE_USE_IMPORT_WORKFLOW)
 GLANCE_ENABLE_QUOTAS=$(trueorfalse True GLANCE_ENABLE_QUOTAS)

 # Flag to set the oslo_policy.enforce_scope. This is used to switch
-# the Image API policies to start checking the scope of token. By Default,
-# this flag is False.
+# This is used to disable the Image API policies scope and new defaults.
+# By Default, it is True.
 # For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
-GLANCE_ENFORCE_SCOPE=$(trueorfalse False GLANCE_ENFORCE_SCOPE)
+GLANCE_ENFORCE_SCOPE=$(trueorfalse True GLANCE_ENFORCE_SCOPE)
+
+# Flag to disable image format inspection on upload
+GLANCE_ENFORCE_IMAGE_FORMAT=$(trueorfalse True GLANCE_ENFORCE_IMAGE_FORMAT)

 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
@@ -168,6 +171,7 @@ function cleanup_glance {
        # Cleanup reserved stores directories
        sudo rm -rf $GLANCE_STAGING_DIR $GLANCE_TASKS_DIR
    fi
+    remove_uwsgi_config "$GLANCE_UWSGI_CONF" "glance-wsgi-api"
 }

 # Set multiple cinder store related config options for each of the cinder store
@@ -288,24 +292,17 @@ function configure_glance_store {

 function configure_glance_quotas {

-    # NOTE(danms): We need to have some of the OS_ things unset in
-    # order to use system scope, which is required for creating these
-    # limits. This is a hack, but I dunno how else to get osc to use
-    # system scope.
+    # Registered limit resources in keystone are system-specific resources.
+    # Make sure we use a system-scoped token to interact with this API.

-    bash -c "unset OS_USERNAME OS_TENANT_NAME OS_PROJECT_NAME;
-        openstack --os-cloud devstack-system-admin registered limit create \
-                --service glance --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL \
-                --region $REGION_NAME image_size_total; \
-        openstack --os-cloud devstack-system-admin registered limit create \
-                --service glance --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL \
-                --region $REGION_NAME image_stage_total; \
-        openstack --os-cloud devstack-system-admin registered limit create \
-                --service glance --default-limit 100 --region $REGION_NAME \
-                image_count_total; \
-        openstack --os-cloud devstack-system-admin registered limit create \
-                --service glance --default-limit 100 --region $REGION_NAME \
-                image_count_uploading"
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL --region $REGION_NAME image_size_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL --region $REGION_NAME image_stage_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit 100 --region $REGION_NAME image_count_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit 100 --region $REGION_NAME image_count_uploading

    # Tell glance to use these limits
    iniset $GLANCE_API_CONF DEFAULT use_keystone_limits True
@@ -316,13 +313,13 @@ function configure_glance_quotas {
    iniset $GLANCE_API_CONF oslo_limit username glance
    iniset $GLANCE_API_CONF oslo_limit auth_type password
    iniset $GLANCE_API_CONF oslo_limit auth_url $KEYSTONE_SERVICE_URI
-    iniset $GLANCE_API_CONF oslo_limit system_scope "'all'"
+    iniset $GLANCE_API_CONF oslo_limit system_scope all
    iniset $GLANCE_API_CONF oslo_limit endpoint_id \
-           $(openstack endpoint list --service glance -f value -c ID)
+           $(openstack --os-cloud devstack-system-admin endpoint list --service glance -f value -c ID)

    # Allow the glance service user to read quotas
-    openstack role add --user glance --user-domain Default --system all \
-              reader
+    openstack --os-cloud devstack-system-admin role add --user glance \
+        --user-domain $SERVICE_DOMAIN_NAME --system all reader
 }

 # configure_glance() - Set config files, create data dirs, etc
@@ -337,6 +334,7 @@ function configure_glance {
    iniset $GLANCE_API_CONF database connection $dburl
    iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_API_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_API_CONF oslo_concurrency lock_path $GLANCE_LOCK_DIR
    iniset $GLANCE_API_CONF paste_deploy flavor keystone+cachemanagement
    configure_keystone_authtoken_middleware $GLANCE_API_CONF glance
@@ -348,6 +346,7 @@ function configure_glance {
    # Only use these if you know what you are doing!  See OSSN-0065
    iniset $GLANCE_API_CONF DEFAULT show_image_direct_url $GLANCE_SHOW_DIRECT_URL
    iniset $GLANCE_API_CONF DEFAULT show_multiple_locations $GLANCE_SHOW_MULTIPLE_LOCATIONS
+    iniset $GLANCE_API_CONF image_format require_image_format_match $GLANCE_ENFORCE_IMAGE_FORMAT

    # Configure glance_store
    configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES
@@ -400,6 +399,7 @@ function configure_glance {
    iniset $GLANCE_CACHE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $GLANCE_CACHE_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_SERVICE_URI
    iniset $GLANCE_CACHE_CONF DEFAULT admin_tenant_name $SERVICE_PROJECT_NAME
    iniset $GLANCE_CACHE_CONF DEFAULT admin_user glance
@@ -437,12 +437,17 @@ function configure_glance {
        iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
        iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
        iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
+        iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url $GLANCE_URL
    fi

-    if [[ "$GLANCE_ENFORCE_SCOPE" == True ]] ; then
+    if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
        iniset $GLANCE_API_CONF oslo_policy enforce_scope true
        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults true
        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac true
+    else
+        iniset $GLANCE_API_CONF oslo_policy enforce_scope false
+        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults false
+        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac false
    fi
 }

@@ -547,7 +552,7 @@ function glance_remote_conf {
 # start_glance_remote_clone() - Clone the regular glance api worker
 function start_glance_remote_clone {
    local glance_remote_conf_dir glance_remote_port remote_data
-    local glance_remote_uwsgi
+    local glance_remote_uwsgi venv

    glance_remote_conf_dir="$(glance_remote_conf "")"
    glance_remote_port=$(get_random_port)
@@ -585,12 +590,16 @@ function start_glance_remote_clone {
    # We need to create the systemd service for the clone, but then
    # change it to include an Environment line to point the WSGI app
    # at the alternate config directory.
+    if [[ "$GLOBAL_VENV" == True ]]; then
+        venv="--venv $DEVSTACK_VENV"
+    fi
    write_uwsgi_user_unit_file devstack@g-api-r.service "$(which uwsgi) \
                               --procname-prefix \
                               glance-api-remote \
-                               --ini $glance_remote_uwsgi" \
+                               --ini $glance_remote_uwsgi \
+                               $venv" \
                               "" "$STACK_USER"
-    iniset -sudo ${SYSTEMD_DIR}/devstack@g-api-r.service \
+    iniadd -sudo ${SYSTEMD_DIR}/devstack@g-api-r.service \
           "Service" "Environment" \
           "OS_GLANCE_CONFIG_DIR=$glance_remote_conf_dir"

@@ -109,12 +109,21 @@ function configure_horizon {
        _horizon_config_set $local_settings "" OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT "True"
    fi

+    if is_service_enabled c-bak; then
+        _horizon_config_set $local_settings OPENSTACK_CINDER_FEATURES enable_backup "True"
+    fi
+
    # Create an empty directory that apache uses as docroot
    sudo mkdir -p $HORIZON_DIR/.blackhole

    local horizon_conf
    horizon_conf=$(apache_site_config_for horizon)

+    local wsgi_venv_config=""
+    if [[ "$GLOBAL_VENV" == "True" ]] ; then
+        wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV"
+    fi
+
    # Configure apache to run horizon
    # Set up the django horizon application to serve via apache/wsgi
    sudo sh -c "sed -e \"
@@ -124,12 +133,13 @@ function configure_horizon {
        s,%APACHE_NAME%,$APACHE_NAME,g;
        s,%DEST%,$DEST,g;
        s,%WEBROOT%,$HORIZON_APACHE_ROOT,g;
+        s,%WSGIPYTHONHOME%,$wsgi_venv_config,g;
    \" $FILES/apache-horizon.template >$horizon_conf"

    if is_ubuntu; then
        disable_apache_site 000-default
        sudo touch $horizon_conf
-    elif is_fedora || is_suse; then
+    elif is_fedora; then
        : # nothing to do
    else
        exit_distro_not_supported "horizon apache configuration"
@@ -163,6 +173,10 @@ function install_horizon {
    # Apache installation, because we mark it NOPRIME
    install_apache_wsgi

+    # Install the memcache library so that horizon can use memcached as its
+    # cache backend
+    pip_install_gr pymemcache
+
    git_clone $HORIZON_REPO $HORIZON_DIR $HORIZON_BRANCH
 }

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Kernel Samepage Merging (KSM)
+# -----------------------------
+
+# Processes that mark their memory as mergeable can share identical memory
+# pages if KSM is enabled. This is particularly useful for nova + libvirt
+# backends but any other setup that marks its memory as mergeable can take
+# advantage. The drawback is there is higher cpu load; however, we tend to
+# be memory bound not cpu bound so enable KSM by default but allow people
+# to opt out if the CPU time is more important to them.
+ENABLE_KSM=$(trueorfalse True ENABLE_KSM)
+ENABLE_KSMTUNED=$(trueorfalse True ENABLE_KSMTUNED)
+function configure_ksm {
+    if [[ $ENABLE_KSMTUNED == "True" ]] ; then
+        install_package "ksmtuned"
+    fi
+    if [[ -f /sys/kernel/mm/ksm/run ]] ; then
+        echo $(bool_to_int ENABLE_KSM) | sudo tee /sys/kernel/mm/ksm/run
+    fi
+}
+
+# Compressed swap (ZSWAP)
+#------------------------
+
+# as noted in the kernel docs https://docs.kernel.org/admin-guide/mm/zswap.html
+# Zswap is a lightweight compressed cache for swap pages.
+# It takes pages that are in the process of being swapped out and attempts
+# to compress them into a dynamically allocated RAM-based memory pool.
+# zswap basically trades CPU cycles for potentially reduced swap I/O.
+# This trade-off can also result in a significant performance improvement
+# if reads from the compressed cache are faster than reads from a swap device.
+
+ENABLE_ZSWAP=$(trueorfalse False ENABLE_ZSWAP)
+# lz4 is very fast although it does not have the best compression
+# zstd has much better compression but more latency
+ZSWAP_COMPRESSOR=${ZSWAP_COMPRESSOR:="lz4"}
+ZSWAP_ZPOOL=${ZSWAP_ZPOOL:="z3fold"}
+function configure_zswap {
+    if [[ $ENABLE_ZSWAP == "True" ]] ; then
+        # Centos 9 stream seems to only support enabling but not run time
+        # tuning so dont try to choose better default on centos
+        if is_ubuntu; then
+            echo ${ZSWAP_COMPRESSOR} | sudo tee /sys/module/zswap/parameters/compressor
+            echo ${ZSWAP_ZPOOL} | sudo tee /sys/module/zswap/parameters/zpool
+        fi
+        echo 1 | sudo tee /sys/module/zswap/parameters/enabled
+        # print curent zswap kernel config
+        sudo grep -R . /sys/module/zswap/parameters || /bin/true
+    fi
+}
+
+ENABLE_SYSCTL_MEM_TUNING=$(trueorfalse False ENABLE_SYSCTL_MEM_TUNING)
+function configure_sysctl_mem_parmaters {
+    if [[ $ENABLE_SYSCTL_MEM_TUNING == "True" ]] ; then
+        # defer write when memory is available
+        sudo sysctl -w vm.dirty_ratio=60
+        sudo sysctl -w vm.dirty_background_ratio=10
+        sudo sysctl -w vm.vfs_cache_pressure=50
+        # assume swap is compressed so on new kernels
+        # give it equal priority as page cache which is
+        # uncompressed. on kernels < 5.8 the max is 100
+        # not 200 so it will strongly prefer swapping.
+        sudo sysctl -w vm.swappiness=100
+        sudo grep -R . /proc/sys/vm/  || /bin/true
+    fi
+}
+
+function configure_host_mem {
+    configure_zswap
+    configure_ksm
+    configure_sysctl_mem_parmaters
+}
+
+ENABLE_SYSCTL_NET_TUNING=$(trueorfalse False ENABLE_SYSCTL_NET_TUNING)
+function configure_sysctl_net_parmaters {
+    if [[ $ENABLE_SYSCTL_NET_TUNING == "True" ]] ; then
+        # detect dead TCP connections after 120 seconds
+        sudo sysctl -w net.ipv4.tcp_keepalive_time=60
+        sudo sysctl -w net.ipv4.tcp_keepalive_intvl=10
+        sudo sysctl -w net.ipv4.tcp_keepalive_probes=6
+        # reudce network latency for new connections
+        sudo sysctl -w net.ipv4.tcp_fastopen=3
+        # print tcp options
+        sudo grep -R . /proc/sys/net/ipv4/tcp* || /bin/true
+        # disable qos by default
+        sudo sysctl -w net.core.default_qdisc=pfifo_fast
+    fi
+}
+
+function configure_host_net {
+    configure_sysctl_net_parmaters
+}
+
+function tune_host {
+    configure_host_mem
+    configure_host_net
+}
@@ -31,7 +31,7 @@ function install_infra {
    local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv"
    [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV
    # We don't care about testing git pbr in the requirements venv.
-    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr
+    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr setuptools
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR

    # Unset the PIP_VIRTUAL_ENV so that PBR does not end up trapped
@@ -9,7 +9,6 @@
 # - ``tls`` file
 # - ``DEST``, ``STACK_USER``
 # - ``FILES``
-# - ``IDENTITY_API_VERSION``
 # - ``BASE_SQL_CONN``
 # - ``SERVICE_HOST``, ``SERVICE_PROTOCOL``
 # - ``S3_SERVICE_PORT`` (template backend only)
@@ -50,9 +49,7 @@ fi
 KEYSTONE_CONF_DIR=${KEYSTONE_CONF_DIR:-/etc/keystone}
 KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
 KEYSTONE_PUBLIC_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-public.ini
-KEYSTONE_ADMIN_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-admin.ini
 KEYSTONE_PUBLIC_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-public
-KEYSTONE_ADMIN_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-admin

 # KEYSTONE_DEPLOY defines how keystone is deployed, allowed values:
 # - mod_wsgi : Run keystone under Apache HTTPd mod_wsgi
@@ -81,21 +78,12 @@ KEYSTONE_RESOURCE_BACKEND=${KEYSTONE_RESOURCE_BACKEND:-sql}
 KEYSTONE_TOKEN_FORMAT=${KEYSTONE_TOKEN_FORMAT:-fernet}
 KEYSTONE_TOKEN_FORMAT=$(echo ${KEYSTONE_TOKEN_FORMAT} | tr '[:upper:]' '[:lower:]')

-# Set Keystone interface configuration
-KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
-KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
-KEYSTONE_AUTH_PORT_INT=${KEYSTONE_AUTH_PORT_INT:-35358}
-KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
-
 # Public facing bits
 KEYSTONE_SERVICE_HOST=${KEYSTONE_SERVICE_HOST:-$SERVICE_HOST}
 KEYSTONE_SERVICE_PORT=${KEYSTONE_SERVICE_PORT:-5000}
 KEYSTONE_SERVICE_PORT_INT=${KEYSTONE_SERVICE_PORT_INT:-5001}
 KEYSTONE_SERVICE_PROTOCOL=${KEYSTONE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}

-# Bind hosts
-KEYSTONE_ADMIN_BIND_HOST=${KEYSTONE_ADMIN_BIND_HOST:-$KEYSTONE_SERVICE_HOST}
-
 # Set the project for service accounts in Keystone
 SERVICE_DOMAIN_NAME=${SERVICE_DOMAIN_NAME:-Default}
 SERVICE_PROJECT_NAME=${SERVICE_PROJECT_NAME:-service}
@@ -106,7 +94,6 @@ SERVICE_TENANT_NAME=${SERVICE_PROJECT_NAME:-service}

 # if we are running with SSL use https protocols
 if is_service_enabled tls-proxy; then
-    KEYSTONE_AUTH_PROTOCOL="https"
    KEYSTONE_SERVICE_PROTOCOL="https"
 fi

@@ -134,6 +121,15 @@ KEYSTONE_PASSWORD_HASH_ROUNDS=${KEYSTONE_PASSWORD_HASH_ROUNDS:-4}
 # Cache settings
 KEYSTONE_ENABLE_CACHE=${KEYSTONE_ENABLE_CACHE:-True}

+# Whether to create a keystone admin endpoint for legacy applications
+KEYSTONE_ADMIN_ENDPOINT=$(trueorfalse False KEYSTONE_ADMIN_ENDPOINT)
+
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the Identity API policies to start checking the scope of token. By Default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+KEYSTONE_ENFORCE_SCOPE=$(trueorfalse False KEYSTONE_ENFORCE_SCOPE)
+
 # Functions
 # ---------

@@ -154,11 +150,8 @@ function cleanup_keystone {
        sudo rm -f $(apache_site_config_for keystone)
    else
        stop_process "keystone"
-        # TODO: remove admin at pike-2
-        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI"
-        remove_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI"
+        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "keystone-wsgi-public"
        sudo rm -f $(apache_site_config_for keystone-wsgi-public)
-        sudo rm -f $(apache_site_config_for keystone-wsgi-admin)
    fi
 }

@@ -171,12 +164,10 @@ function _config_keystone_apache_wsgi {
    local keystone_certfile=""
    local keystone_keyfile=""
    local keystone_service_port=$KEYSTONE_SERVICE_PORT
-    local keystone_auth_port=$KEYSTONE_AUTH_PORT
    local venv_path=""

    if is_service_enabled tls-proxy; then
        keystone_service_port=$KEYSTONE_SERVICE_PORT_INT
-        keystone_auth_port=$KEYSTONE_AUTH_PORT_INT
    fi
    if [[ ${USE_VENV} = True ]]; then
        venv_path="python-path=${PROJECT_VENV["keystone"]}/lib/$(python_version)/site-packages"
@@ -185,7 +176,6 @@ function _config_keystone_apache_wsgi {
    sudo cp $FILES/apache-keystone.template $keystone_apache_conf
    sudo sed -e "
        s|%PUBLICPORT%|$keystone_service_port|g;
-        s|%ADMINPORT%|$keystone_auth_port|g;
        s|%APACHE_NAME%|$APACHE_NAME|g;
        s|%SSLLISTEN%|$keystone_ssl_listen|g;
        s|%SSLENGINE%|$keystone_ssl|g;
@@ -223,22 +213,17 @@ function configure_keystone {
    iniset_rpc_backend keystone $KEYSTONE_CONF oslo_messaging_notifications

    local service_port=$KEYSTONE_SERVICE_PORT
-    local auth_port=$KEYSTONE_AUTH_PORT

    if is_service_enabled tls-proxy; then
        # Set the service ports for a proxy to take the originals
        service_port=$KEYSTONE_SERVICE_PORT_INT
-        auth_port=$KEYSTONE_AUTH_PORT_INT
    fi

-    # Override the endpoints advertised by keystone (the public_endpoint and
-    # admin_endpoint) so that clients use the correct endpoint. By default, the
-    # keystone server uses the public_port and admin_port which isn't going to
-    # work when you want to use a different port (in the case of proxy), or you
-    # don't want the port (in the case of putting keystone on a path in
-    # apache).
+    # Override the endpoints advertised by keystone so that clients use the correct
+    # endpoint. By default, the keystone server uses the public_port which isn't
+    # going to work when you want to use a different port (in the case of proxy),
+    # or you don't want the port (in the case of putting keystone on a path in apache).
    iniset $KEYSTONE_CONF DEFAULT public_endpoint $KEYSTONE_SERVICE_URI
-    iniset $KEYSTONE_CONF DEFAULT admin_endpoint $KEYSTONE_AUTH_URI

    if [[ "$KEYSTONE_TOKEN_FORMAT" != "" ]]; then
        iniset $KEYSTONE_CONF token provider $KEYSTONE_TOKEN_FORMAT
@@ -261,7 +246,6 @@ function configure_keystone {
        _config_keystone_apache_wsgi
    else # uwsgi
        write_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI" "/identity"
-        write_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI" "/identity_admin"
    fi

    iniset $KEYSTONE_CONF DEFAULT max_token_size 16384
@@ -281,6 +265,16 @@ function configure_keystone {
        iniset $KEYSTONE_CONF security_compliance lockout_duration $KEYSTONE_LOCKOUT_DURATION
        iniset $KEYSTONE_CONF security_compliance unique_last_password_count $KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT
    fi
+
+    iniset $KEYSTONE_CONF oslo_policy policy_file policy.yaml
+
+    if [[ "$KEYSTONE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $KEYSTONE_CONF oslo_policy enforce_scope true
+        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults true
+    else
+        iniset $KEYSTONE_CONF oslo_policy enforce_scope false
+        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults false
+    fi
 }

 # create_keystone_accounts() - Sets up common required keystone accounts
@@ -303,20 +297,28 @@ function configure_keystone {
 # admins               admin            admin                 admin
 # nonadmins            demo, alt_demo   member, anotherrole   demo, alt_demo

+# System               User            Roles
+# ------------------------------------------------------------------
+# all                  admin           admin
+# all                  system_reader   reader
+# all                  system_member   member
+

 # Migrated from keystone_data.sh
 function create_keystone_accounts {

    # The keystone bootstrapping process (performed via keystone-manage
-    # bootstrap) creates an admin user, admin role, member role, and admin
+    # bootstrap) creates an admin user and an admin
    # project. As a sanity check we exercise the CLI to retrieve the IDs for
    # these values.
    local admin_project
    admin_project=$(openstack project show "admin" -f value -c id)
    local admin_user
    admin_user=$(openstack user show "admin" -f value -c id)
+    # These roles are also created during bootstrap but we don't need their IDs
    local admin_role="admin"
    local member_role="member"
+    local reader_role="reader"

    async_run ks-domain-role get_or_add_user_domain_role $admin_role $admin_user default

@@ -352,21 +354,53 @@ function create_keystone_accounts {
    async_wait ks-{domain-role,domain,project,service,reseller,anotherrole}

    async_run ks-demo-member get_or_add_user_project_role $member_role $demo_user $demo_project
+
    async_run ks-demo-admin get_or_add_user_project_role $admin_role $admin_user $demo_project
    async_run ks-demo-another get_or_add_user_project_role $another_role $demo_user $demo_project
    async_run ks-demo-invis get_or_add_user_project_role $member_role $demo_user $invis_project

-    # alt_demo
+    # Create a user to act as a reader on project demo
+    local demo_reader
+    demo_reader=$(get_or_create_user "demo_reader" \
+        "$ADMIN_PASSWORD" "default" "demo_reader@example.com")
+
+    async_run ks-demo-reader get_or_add_user_project_role $reader_role $demo_reader $demo_project
+
+    # Create a different project called alt_demo
    local alt_demo_project
    alt_demo_project=$(get_or_create_project "alt_demo" default)
+    # Create a user to act as member, admin and anotherrole on project alt_demo
    local alt_demo_user
    alt_demo_user=$(get_or_create_user "alt_demo" \
        "$ADMIN_PASSWORD" "default" "alt_demo@example.com")

-    async_run ks-alt-member get_or_add_user_project_role $member_role $alt_demo_user $alt_demo_project
-    async_run ks-alt-admin get_or_add_user_project_role $admin_role $admin_user $alt_demo_project
+    async_run ks-alt-admin get_or_add_user_project_role $admin_role $alt_demo_user $alt_demo_project
    async_run ks-alt-another get_or_add_user_project_role $another_role $alt_demo_user $alt_demo_project

+    # Create another user to act as a member on project alt_demo
+    local alt_demo_member
+    alt_demo_member=$(get_or_create_user "alt_demo_member" \
+        "$ADMIN_PASSWORD" "default" "alt_demo_member@example.com")
+    async_run ks-alt-member-user get_or_add_user_project_role $member_role $alt_demo_member $alt_demo_project
+
+    # Create another user to act as a reader on project alt_demo
+    local alt_demo_reader
+    alt_demo_reader=$(get_or_create_user "alt_demo_reader" \
+        "$ADMIN_PASSWORD" "default" "alt_demo_reader@example.com")
+    async_run ks-alt-reader-user get_or_add_user_project_role $reader_role $alt_demo_reader $alt_demo_project
+
+    # Create two users, give one the member role on the system and the other the
+    # reader role on the system. These two users model system-member and
+    # system-reader personas. The admin user already has the admin role on the
+    # system and we can re-use this user as a system-admin.
+    system_member_user=$(get_or_create_user "system_member" \
+        "$ADMIN_PASSWORD" "default" "system_member@example.com")
+    async_run ks-system-member get_or_add_user_system_role $member_role $system_member_user "all"
+
+    system_reader_user=$(get_or_create_user "system_reader" \
+        "$ADMIN_PASSWORD" "default" "system_reader@example.com")
+    async_run ks-system-reader get_or_add_user_system_role $reader_role $system_reader_user "all"
+
    # groups
    local admin_group
    admin_group=$(get_or_create_group "admins" \
@@ -381,8 +415,9 @@ function create_keystone_accounts {
    async_run ks-group-anotheralt get_or_add_group_project_role $another_role $non_admin_group $alt_demo_project
    async_run ks-group-admin get_or_add_group_project_role $admin_role $admin_group $admin_project

-    async_wait ks-demo-{member,admin,another,invis}
-    async_wait ks-alt-{member,admin,another}
+    async_wait ks-demo-{member,admin,another,invis,reader}
+    async_wait ks-alt-{admin,another,member-user,reader-user}
+    async_wait ks-system-{member,reader}
    async_wait ks-group-{memberdemo,anotherdemo,memberalt,anotheralt,admin}

    if is_service_enabled ldap; then
@@ -518,7 +553,7 @@ function install_keystone {
 function start_keystone {
    # Get right service port for testing
    local service_port=$KEYSTONE_SERVICE_PORT
-    local auth_protocol=$KEYSTONE_AUTH_PROTOCOL
+    local auth_protocol=$KEYSTONE_SERVICE_PROTOCOL
    if is_service_enabled tls-proxy; then
        service_port=$KEYSTONE_SERVICE_PORT_INT
        auth_protocol="http"
@@ -537,7 +572,7 @@ function start_keystone {
    # unencryted traffic at this point.
    # If running in Apache, use the path rather than port.

-    local service_uri=$auth_protocol://$KEYSTONE_SERVICE_HOST/identity/v$IDENTITY_API_VERSION/
+    local service_uri=$auth_protocol://$KEYSTONE_SERVICE_HOST/identity/v3/

    if ! wait_for_service $SERVICE_TIMEOUT $service_uri; then
        die $LINENO "keystone did not start"
@@ -546,7 +581,6 @@ function start_keystone {
    # Start proxies if enabled
    if is_service_enabled tls-proxy; then
        start_tls_proxy keystone-service '*' $KEYSTONE_SERVICE_PORT $KEYSTONE_SERVICE_HOST $KEYSTONE_SERVICE_PORT_INT
-        start_tls_proxy keystone-auth '*' $KEYSTONE_AUTH_PORT $KEYSTONE_AUTH_HOST $KEYSTONE_AUTH_PORT_INT
    fi

    # (re)start memcached to make sure we have a clean memcache.
@@ -567,11 +601,8 @@ function stop_keystone {
 # This function uses the following GLOBAL variables:
 # - ``KEYSTONE_BIN_DIR``
 # - ``ADMIN_PASSWORD``
-# - ``IDENTITY_API_VERSION``
 # - ``REGION_NAME``
-# - ``KEYSTONE_SERVICE_PROTOCOL``
-# - ``KEYSTONE_SERVICE_HOST``
-# - ``KEYSTONE_SERVICE_PORT``
+# - ``KEYSTONE_SERVICE_URI``
 function bootstrap_keystone {
    $KEYSTONE_BIN_DIR/keystone-manage bootstrap \
        --bootstrap-username admin \
@@ -580,8 +611,16 @@ function bootstrap_keystone {
        --bootstrap-role-name admin \
        --bootstrap-service-name keystone \
        --bootstrap-region-id "$REGION_NAME" \
-        --bootstrap-admin-url "$KEYSTONE_AUTH_URI" \
        --bootstrap-public-url "$KEYSTONE_SERVICE_URI"
+    if [ "$KEYSTONE_ADMIN_ENDPOINT" == "True" ]; then
+        openstack endpoint create --region "$REGION_NAME" \
+            --os-username admin \
+            --os-user-domain-id default \
+            --os-password "$ADMIN_PASSWORD" \
+            --os-project-name admin \
+            --os-project-domain-id default \
+            keystone admin "$KEYSTONE_SERVICE_URI"
+    fi
 }

 # create_ldap_domain() - Create domain file and initialize domain with a user
@@ -33,16 +33,12 @@ LDAP_SERVICE_NAME=slapd

 if is_ubuntu; then
    LDAP_OLCDB_NUMBER=1
+    LDAP_OLCDB_TYPE=mdb
    LDAP_ROOTPW_COMMAND=replace
 elif is_fedora; then
    LDAP_OLCDB_NUMBER=2
+    LDAP_OLCDB_TYPE=hdb
    LDAP_ROOTPW_COMMAND=add
-elif is_suse; then
-    # SUSE has slappasswd in /usr/sbin/
-    PATH=$PATH:/usr/sbin/
-    LDAP_OLCDB_NUMBER=1
-    LDAP_ROOTPW_COMMAND=add
-    LDAP_SERVICE_NAME=ldap
 fi


@@ -56,6 +52,7 @@ function _ldap_varsubst {
    local slappass=$2
    sed -e "
        s|\${LDAP_OLCDB_NUMBER}|$LDAP_OLCDB_NUMBER|
+        s|\${LDAP_OLCDB_TYPE}|$LDAP_OLCDB_TYPE|
        s|\${SLAPPASS}|$slappass|
        s|\${LDAP_ROOTPW_COMMAND}|$LDAP_ROOTPW_COMMAND|
        s|\${BASE_DC}|$LDAP_BASE_DC|
@@ -72,8 +69,6 @@ function cleanup_ldap {
        sudo rm -rf /etc/ldap/ldap.conf /var/lib/ldap
    elif is_fedora; then
        sudo rm -rf /etc/openldap /var/lib/ldap
-    elif is_suse; then
-        sudo rm -rf /var/lib/ldap
    fi
 }

@@ -122,11 +117,6 @@ function install_ldap {
        configure_ldap
    elif is_fedora; then
        start_ldap
-    elif is_suse; then
-        _ldap_varsubst $FILES/ldap/suse-base-config.ldif.in >$tmp_ldap_dir/suse-base-config.ldif
-        sudo slapadd -F /etc/openldap/slapd.d/ -bcn=config -l $tmp_ldap_dir/suse-base-config.ldif
-        sudo sed -i '/^OPENLDAP_START_LDAPI=/s/"no"/"yes"/g' /etc/sysconfig/openldap
-        start_ldap
    fi

    echo "LDAP_PASSWORD is $LDAP_PASSWORD"
@@ -157,7 +147,7 @@ function configure_ldap {
    slapd slapd/dump_database_destdir string /var/backups/slapd-VERSION
    slapd slapd/domain string Users
    slapd shared/organization string $LDAP_DOMAIN
-    slapd slapd/backend string HDB
+    slapd slapd/backend string ${LDAP_OLCDB_TYPE^^}
    slapd slapd/purge_database boolean true
    slapd slapd/move_old_database boolean true
    slapd slapd/allow_ldap_v2 boolean false
@@ -38,6 +38,7 @@ GITDIR["oslo.config"]=$DEST/oslo.config
 GITDIR["oslo.context"]=$DEST/oslo.context
 GITDIR["oslo.db"]=$DEST/oslo.db
 GITDIR["oslo.i18n"]=$DEST/oslo.i18n
+GITDIR["oslo.limit"]=$DEST/oslo.limit
 GITDIR["oslo.log"]=$DEST/oslo.log
 GITDIR["oslo.messaging"]=$DEST/oslo.messaging
 GITDIR["oslo.middleware"]=$DEST/oslo.middleware
@@ -102,6 +103,7 @@ function install_libs {
    _install_lib_from_source "oslo.context"
    _install_lib_from_source "oslo.db"
    _install_lib_from_source "oslo.i18n"
+    _install_lib_from_source "oslo.limit"
    _install_lib_from_source "oslo.log"
    _install_lib_from_source "oslo.messaging"
    _install_lib_from_source "oslo.middleware"
@@ -53,28 +53,10 @@ function _remove_lvm_volume_group {
    sudo vgremove -f $vg
 }

-# _clean_lvm_backing_file() removes the backing file of the
-# volume group
-#
-# Usage: _clean_lvm_backing_file() $backing_file
-function _clean_lvm_backing_file {
-    local backing_file=$1
-
-    # If the backing physical device is a loop device, it was probably setup by DevStack
-    if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
-        local vg_dev
-        vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}')
-        if [[ -n "$vg_dev" ]]; then
-            sudo losetup -d $vg_dev
-        fi
-        rm -f $backing_file
-    fi
-}
-
 # clean_lvm_volume_group() cleans up the volume group and removes the
 # backing file
 #
-# Usage: clean_lvm_volume_group $vg
+# Usage: clean_lvm_volume_group() $vg
 function clean_lvm_volume_group {
    local vg=$1

@@ -83,11 +65,22 @@ function clean_lvm_volume_group {
    # if there is no logical volume left, it's safe to attempt a cleanup
    # of the backing file
    if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
-        _clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX
+        local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX
+
+        if [[ -n "$vg$BACKING_FILE_SUFFIX" ]] && \
+           [[ -e "/etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service" ]]; then
+            sudo systemctl disable --now $vg$BACKING_FILE_SUFFIX.service
+            sudo rm -f /etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service
+            sudo systemctl daemon-reload
+        fi
+
+        # If the backing physical device is a loop device, it was probably setup by DevStack
+        if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
+            rm -f $backing_file
+        fi
    fi
 }

-
 # _create_lvm_volume_group creates default volume group
 #
 # Usage: _create_lvm_volume_group() $vg $size
@@ -106,8 +99,20 @@ function _create_lvm_volume_group {
            directio="--direct-io=on"
        fi

+        # Only create systemd service if it doesn't already exists
+        if [[ ! -e "/etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service" ]]; then
+            sed -e "
+                s|%DIRECTIO%|${directio}|g;
+                s|%BACKING_FILE%|${backing_file}|g;
+            " $FILES/lvm-backing-file.template | sudo tee \
+                /etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service
+
+            sudo systemctl daemon-reload
+            sudo systemctl enable --now $vg$BACKING_FILE_SUFFIX.service
+        fi
+
        local vg_dev
-        vg_dev=$(sudo losetup -f --show $directio $backing_file)
+        vg_dev=$(sudo losetup --associated $backing_file -O NAME -n)

        # Only create volume group if it doesn't already exist
        if ! sudo vgs $vg; then
@@ -124,19 +129,25 @@ function init_lvm_volume_group {
    local vg=$1
    local size=$2

-    # Start the tgtd service on Fedora and SUSE if tgtadm is used
-    if  is_fedora || is_suse  && [[ "$CINDER_ISCSI_HELPER" = "tgtadm" ]]; then
+    # Start the tgtd service on Fedora if tgtadm is used
+    if  is_fedora; then
        start_service tgtd
    fi

    # Start with a clean volume group
    _create_lvm_volume_group $vg $size

-    # Remove iscsi targets
-    if [ "$CINDER_ISCSI_HELPER" = "lioadm" ]; then
-        sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
-    else
-        sudo tgtadm --op show --mode target | awk '/Target/ {print $3}' | sudo xargs -r -n1 tgt-admin --delete
+    if is_service_enabled cinder; then
+        # Remove iscsi targets
+        if [ "$CINDER_TARGET_HELPER" = "lioadm" ]; then
+            sudo cinder-rtstool get-targets | sudo xargs -rn 1 cinder-rtstool delete
+        elif [ "$CINDER_TARGET_HELPER" = "tgtadm" ]; then
+            sudo tgtadm --op show --mode target | awk '/Target/ {print $3}' | sudo xargs -r -n1 tgt-admin --delete
+        elif [ "$CINDER_TARGET_HELPER" = "nvmet" ]; then
+            # If we don't disconnect everything vgremove will block
+            sudo nvme disconnect-all
+            sudo nvmetcli clear
+        fi
    fi
    _clean_lvm_volume_group $vg
 }
@@ -189,7 +200,7 @@ function set_lvm_filter {
    filter_string=$filter_string$filter_suffix

    clean_lvm_filter
-    sudo sed -i "/# global_filter = \[*\]/a\    $global_filter$filter_string" /etc/lvm/lvm.conf
+    sudo sed -i "/# global_filter = \[.*\]/a\        $filter_string" /etc/lvm/lvm.conf
    echo_summary "set lvm.conf device global_filter to: $filter_string"
 }

@@ -13,7 +13,7 @@ Plugin specific configuration variables should be in this file.

 functions
 ---------
-``lib/neutron-legacy`` calls the following functions when the ``$Q_PLUGIN`` is enabled
+``lib/neutron`` calls the following functions when the ``$Q_PLUGIN`` is enabled

 * ``neutron_plugin_create_nova_conf`` :
  optionally set options in nova_conf
@@ -67,7 +67,7 @@ function has_neutron_plugin_security_group {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -72,7 +72,7 @@ function has_neutron_plugin_security_group {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -97,7 +97,7 @@ function neutron_plugin_setup_interface_driver {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -67,7 +67,7 @@ function neutron_plugin_configure_common {
    Q_PLUGIN_CLASS="ml2"
    # The ML2 plugin delegates L3 routing/NAT functionality to
    # the L3 service plugin which must therefore be specified.
-    _neutron_service_plugin_class_add $ML2_L3_PLUGIN
+    neutron_service_plugin_class_add $ML2_L3_PLUGIN
 }

 function neutron_plugin_configure_service {
@@ -111,20 +111,13 @@ function neutron_plugin_configure_service {
            fi
        fi
    fi
-    # REVISIT(rkukura): Setting firewall_driver here for
-    # neutron.agent.securitygroups_rpc.is_firewall_enabled() which is
-    # used in the server, in case no L2 agent is configured on the
-    # server's node. If an L2 agent is configured, this will get
-    # overridden with the correct driver. The ml2 plugin should
-    # instead use its own config variable to indicate whether security
-    # groups is enabled, and that will need to be set here instead.
-    if [[ "$Q_USE_SECGROUP" == "True" ]]; then
-        iniset /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver neutron.agent.not.a.real.FirewallDriver
-    else
-        iniset /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver neutron.agent.firewall.NoopFirewallDriver
-    fi
+    populate_ml2_config /$Q_PLUGIN_CONF_FILE securitygroup enable_security_group=$Q_USE_SECGROUP

    populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 mechanism_drivers=$Q_ML2_PLUGIN_MECHANISM_DRIVERS
+    if [[ "$Q_ML2_PLUGIN_MECHANISM_DRIVERS" == *"linuxbridge"* ]]; then
+        iniset $NEUTRON_CONF experimental linuxbridge True
+    fi
+    populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 overlay_ip_version=$TUNNEL_IP_VERSION

    if [[ -n "$Q_ML2_PLUGIN_TYPE_DRIVERS" ]]; then
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 type_drivers=$Q_ML2_PLUGIN_TYPE_DRIVERS
@@ -156,5 +149,9 @@ function has_neutron_plugin_security_group {
    return 0
 }

+function configure_qos_ml2 {
+    neutron_ml2_extension_driver_add "qos"
+}
+
 # Restore xtrace
 $_XTRACE_NEUTRON_ML2
@@ -68,7 +68,7 @@ function neutron_plugin_setup_interface_driver {
 }

 function neutron_plugin_check_adv_test_requirements {
-    is_service_enabled q-agt && is_service_enabled q-dhcp && return 0
+    is_service_enabled q-agt neutron-agent && is_service_enabled q-dhcp neutron-dhcp && return 0
 }

 # Restore xtrace
@@ -24,11 +24,6 @@ source ${TOP_DIR}/lib/neutron_plugins/openvswitch_agent
 # Load devstack ovs compliation and loading functions
 source ${TOP_DIR}/lib/neutron_plugins/ovs_source

-# Defaults
-# --------
-
-Q_BUILD_OVS_FROM_GIT=$(trueorfalse True Q_BUILD_OVS_FROM_GIT)
-
 # Set variables for building OVN from source
 OVN_REPO=${OVN_REPO:-https://github.com/ovn-org/ovn.git}
 OVN_REPO_NAME=$(basename ${OVN_REPO} | cut -f1 -d'.')
@@ -74,6 +69,9 @@ OVN_UUID=${OVN_UUID:-}
 # unless the distro kernel includes ovs+conntrack support.
 OVN_BUILD_MODULES=$(trueorfalse False OVN_BUILD_MODULES)
 OVN_BUILD_FROM_SOURCE=$(trueorfalse False OVN_BUILD_FROM_SOURCE)
+if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+    Q_BUILD_OVS_FROM_GIT=True
+fi

 # Whether or not to install the ovs python module from ovs source.  This can be
 # used to test and validate new ovs python features.  This should only be used
@@ -93,16 +91,23 @@ OVN_GENEVE_OVERHEAD=${OVN_GENEVE_OVERHEAD:-38}
 # http://www.openvswitch.org/support/dist-docs/ovs-appctl.8.txt
 OVN_DBS_LOG_LEVEL=${OVN_DBS_LOG_LEVEL:-info}

+# OVN metadata agent configuration
 OVN_META_CONF=$NEUTRON_CONF_DIR/neutron_ovn_metadata_agent.ini
 OVN_META_DATA_HOST=${OVN_META_DATA_HOST:-$(ipv6_unquote $SERVICE_HOST)}

+# OVN agent configuration
+OVN_AGENT_CONF=$NEUTRON_CONF_DIR/plugins/ml2/ovn_agent.ini
+OVN_AGENT_EXTENSIONS=${OVN_AGENT_EXTENSIONS:-}
+
 # If True (default) the node will be considered a gateway node.
 ENABLE_CHASSIS_AS_GW=$(trueorfalse True ENABLE_CHASSIS_AS_GW)
 OVN_L3_CREATE_PUBLIC_NETWORK=$(trueorfalse True OVN_L3_CREATE_PUBLIC_NETWORK)

 export OVSDB_SERVER_LOCAL_HOST=$SERVICE_LOCAL_HOST
+TUNNEL_IP=$TUNNEL_ENDPOINT_IP
 if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
    OVSDB_SERVER_LOCAL_HOST=[$OVSDB_SERVER_LOCAL_HOST]
+    TUNNEL_IP=[$TUNNEL_IP]
 fi

 OVN_IGMP_SNOOPING_ENABLE=$(trueorfalse False OVN_IGMP_SNOOPING_ENABLE)
@@ -119,13 +124,20 @@ OVS_SCRIPTDIR=$OVS_SHAREDIR/scripts
 OVS_DATADIR=$DATA_DIR/ovs
 OVS_SYSCONFDIR=${OVS_SYSCONFDIR:-$OVS_PREFIX/etc/openvswitch}

-OVN_DATADIR=$DATA_DIR/ovn
+if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+    OVN_DATADIR=$DATA_DIR/ovn
+else
+    # When using OVN from packages, the data dir for OVN DBs is
+    # /var/lib/ovn
+    OVN_DATADIR=/var/lib/ovn
+fi
 OVN_SHAREDIR=$OVS_PREFIX/share/ovn
 OVN_SCRIPTDIR=$OVN_SHAREDIR/scripts
 OVN_RUNDIR=$OVS_PREFIX/var/run/ovn

 NEUTRON_OVN_BIN_DIR=$(get_python_exec_prefix)
 NEUTRON_OVN_METADATA_BINARY="neutron-ovn-metadata-agent"
+NEUTRON_OVN_AGENT_BINARY="neutron-ovn-agent"

 STACK_GROUP="$( id --group --name "$STACK_USER" )"

@@ -165,12 +177,23 @@ Q_LOG_DRIVER_LOG_BASE=${Q_LOG_DRIVER_LOG_BASE:-acl_log_meter}
 # Utility Functions
 # -----------------

+function wait_for_db_file {
+    local count=0
+    while [ ! -f $1 ]; do
+        sleep 1
+        count=$((count+1))
+        if [ "$count" -gt 40 ]; then
+            die $LINENO "DB File $1 not found"
+        fi
+    done
+}
+
 function wait_for_sock_file {
    local count=0
    while [ ! -S $1 ]; do
        sleep 1
        count=$((count+1))
-        if [ "$count" -gt 5 ]; then
+        if [ "$count" -gt 40 ]; then
            die $LINENO "Socket $1 not found"
        fi
    done
@@ -227,11 +250,12 @@ function _run_process {
    local cmd="$2"
    local stop_cmd="$3"
    local group=$4
-    local user=${5:-$STACK_USER}
+    local user=$5
+    local rundir=${6:-$OVS_RUNDIR}

    local systemd_service="devstack@$service.service"
    local unit_file="$SYSTEMD_DIR/$systemd_service"
-    local environment="OVN_RUNDIR=$OVS_RUNDIR OVN_DBDIR=$OVN_DATADIR OVN_LOGDIR=$LOGDIR OVS_RUNDIR=$OVS_RUNDIR OVS_DBDIR=$OVS_DATADIR OVS_LOGDIR=$LOGDIR"
+    local environment="OVN_RUNDIR=$OVN_RUNDIR OVN_DBDIR=$OVN_DATADIR OVN_LOGDIR=$LOGDIR OVS_RUNDIR=$OVS_RUNDIR OVS_DBDIR=$OVS_DATADIR OVS_LOGDIR=$LOGDIR"

    echo "Starting $service executed command": $cmd

@@ -247,9 +271,14 @@ function _run_process {

    _start_process $systemd_service

-    local testcmd="test -e $OVS_RUNDIR/$service.pid"
+    local testcmd="test -e $rundir/$service.pid"
    test_with_retry "$testcmd" "$service did not start" $SERVICE_TIMEOUT 1
-    sudo ovs-appctl -t $service vlog/set console:off syslog:info file:info
+    local service_ctl_file
+    service_ctl_file=$(ls $rundir | grep $service | grep ctl)
+    if [ -z "$service_ctl_file" ]; then
+        die $LINENO "ctl file for service $service is not present."
+    fi
+    sudo ovs-appctl -t $rundir/$service_ctl_file vlog/set console:off syslog:info file:info
 }

 function clone_repository {
@@ -265,7 +294,7 @@ function clone_repository {
 function create_public_bridge {
    # Create the public bridge that OVN will use
    sudo ovs-vsctl --may-exist add-br $PUBLIC_BRIDGE -- set bridge $PUBLIC_BRIDGE protocols=OpenFlow13,OpenFlow15
-    sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=$PHYSICAL_NETWORK:$PUBLIC_BRIDGE
+    sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=${OVN_BRIDGE_MAPPINGS}
    _configure_public_network_connectivity
 }

@@ -288,16 +317,13 @@ function _disable_libvirt_apparmor {

 # compile_ovn() - Compile OVN from source and load needed modules
 #                 Accepts three parameters:
-#                   - first optional is False by default and means that
-#                     modules are built and installed.
-#                   - second optional parameter defines prefix for
+#                   - first optional parameter defines prefix for
 #                     ovn compilation
-#                   - third optional parameter defines localstatedir for
+#                   - second optional parameter defines localstatedir for
 #                     ovn single machine runtime
 function compile_ovn {
-    local build_modules=${1:-False}
-    local prefix=$2
-    local localstatedir=$3
+    local prefix=$1
+    local localstatedir=$2

    if [ -n "$prefix" ]; then
        prefix="--prefix=$prefix"
@@ -314,8 +340,24 @@ function compile_ovn {
        ./boot.sh
    fi

+    # NOTE(mnaser): OVN requires that you build using the OVS from the
+    #               submodule.
+    #
+    #               https://github.com/ovn-org/ovn/blob/3fb397b63663297acbcbf794e1233951222ae5af/Documentation/intro/install/general.rst#bootstrapping
+    #               https://github.com/ovn-org/ovn/issues/128
+    git submodule update --init
+    pushd ovs
+    if [ ! -f configure ] ; then
+        ./boot.sh
+    fi
    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
-        ./configure --with-ovs-source=$DEST/$OVS_REPO_NAME $prefix $localstatedir
+        ./configure
+    fi
+    make -j$(($(nproc) + 1))
+    popd
+
+    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
+        ./configure $prefix $localstatedir
    fi
    make -j$(($(nproc) + 1))
    sudo make install
@@ -328,7 +370,7 @@ function compile_ovn {

 # OVN service sanity check
 function ovn_sanity_check {
-    if is_service_enabled q-agt neutron-agt; then
+    if is_service_enabled q-agt neutron-agent; then
        die $LINENO "The q-agt/neutron-agt service must be disabled with OVN."
    elif is_service_enabled q-l3 neutron-l3; then
        die $LINENO "The q-l3/neutron-l3 service must be disabled with OVN."
@@ -341,11 +383,6 @@ function ovn_sanity_check {

 # install_ovn() - Collect source and prepare
 function install_ovn {
-    if [[ "$Q_BUILD_OVS_FROM_GIT" == "False" ]]; then
-        echo "Installation of OVS from source disabled."
-        return 0
-    fi
-
    echo "Installing OVN and dependent packages"

    # Check the OVN configuration
@@ -356,10 +393,6 @@ function install_ovn {

    sudo mkdir -p $OVS_RUNDIR
    sudo chown $(whoami) $OVS_RUNDIR
-    # NOTE(lucasagomes): To keep things simpler, let's reuse the same
-    # RUNDIR for both OVS and OVN. This way we avoid having to specify the
-    # --db option in the ovn-{n,s}bctl commands while playing with DevStack
-    sudo ln -s $OVS_RUNDIR $OVN_RUNDIR

    if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
        # If OVS is already installed, remove it, because we're about to
@@ -375,7 +408,7 @@ function install_ovn {

        compile_ovs $OVN_BUILD_MODULES
        if use_new_ovn_repository; then
-            compile_ovn $OVN_BUILD_MODULES
+            compile_ovn
        fi

        sudo mkdir -p $OVS_PREFIX/var/log/openvswitch
@@ -383,6 +416,8 @@ function install_ovn {
        sudo mkdir -p $OVS_PREFIX/var/log/ovn
        sudo chown $(whoami) $OVS_PREFIX/var/log/ovn
    else
+        # Load fixup_ovn_centos
+        source ${TOP_DIR}/tools/fixup_stuff.sh
        fixup_ovn_centos
        install_package $(get_packages openvswitch)
        install_package $(get_packages ovn)
@@ -448,7 +483,7 @@ function filter_network_api_extensions {
 function configure_ovn_plugin {
    echo "Configuring Neutron for OVN"

-    if is_service_enabled q-svc ; then
+    if is_service_enabled q-svc neutron-api; then
        filter_network_api_extensions
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2_type_geneve max_header_size=$OVN_GENEVE_OVERHEAD
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_connection="$OVN_NB_REMOTE"
@@ -472,7 +507,9 @@ function configure_ovn_plugin {
            inicomment /$Q_PLUGIN_CONF_FILE network_log local_output_log_base="$Q_LOG_DRIVER_LOG_BASE"
        fi

-        if is_service_enabled q-ovn-metadata-agent; then
+        if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
+        elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
        else
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=False
@@ -493,7 +530,9 @@ function configure_ovn_plugin {
    fi

    if is_service_enabled n-api-meta ; then
-        if is_service_enabled q-ovn-metadata-agent ; then
+        if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
+            iniset $NOVA_CONF neutron service_metadata_proxy True
+        elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then
            iniset $NOVA_CONF neutron service_metadata_proxy True
        fi
    fi
@@ -526,29 +565,42 @@ function configure_ovn {
    fi

    # Metadata
-    if is_service_enabled q-ovn-metadata-agent && is_service_enabled ovn-controller; then
+    local sample_file=""
+    local config_file=""
+    if is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]] && is_service_enabled ovn-controller; then
+        sample_file=$NEUTRON_DIR/etc/neutron/plugins/ml2/ovn_agent.ini.sample
+        config_file=$OVN_AGENT_CONF
+    elif is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent && is_service_enabled ovn-controller; then
+        sample_file=$NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample
+        config_file=$OVN_META_CONF
+    fi
+    if [ -n "$config_file" ]; then
        sudo install -d -o $STACK_USER $NEUTRON_CONF_DIR

        mkdir -p $NEUTRON_DIR/etc/neutron/plugins/ml2
        (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)

-        cp $NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample $OVN_META_CONF
-        configure_root_helper_options $OVN_META_CONF
+        cp $sample_file $config_file
+        configure_root_helper_options $config_file

-        iniset $OVN_META_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
-        iniset $OVN_META_CONF DEFAULT nova_metadata_host $OVN_META_DATA_HOST
-        iniset $OVN_META_CONF DEFAULT metadata_workers $API_WORKERS
-        iniset $OVN_META_CONF DEFAULT state_path $NEUTRON_STATE_PATH
-        iniset $OVN_META_CONF ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
-        iniset $OVN_META_CONF ovn ovn_sb_connection $OVN_SB_REMOTE
+        iniset $config_file DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
+        iniset $config_file DEFAULT nova_metadata_host $OVN_META_DATA_HOST
+        iniset $config_file DEFAULT metadata_workers $API_WORKERS
+        iniset $config_file DEFAULT state_path $DATA_DIR/neutron
+        iniset $config_file ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
+        iniset $config_file ovn ovn_sb_connection $OVN_SB_REMOTE
        if is_service_enabled tls-proxy; then
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_ca_cert $INT_CA_DIR/ca-chain.pem
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_certificate $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_private_key $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key
        fi
+        if [[ $config_file == $OVN_AGENT_CONF ]]; then
+            iniset $config_file agent extensions $OVN_AGENT_EXTENSIONS
+            iniset $config_file ovn ovn_nb_connection $OVN_NB_REMOTE
+        fi
    fi
 }

@@ -561,14 +613,20 @@ function init_ovn {
    # create new ones on each devstack run.

    _disable_libvirt_apparmor
+    local mkdir_cmd="mkdir -p ${OVN_DATADIR}"

-    mkdir -p $OVN_DATADIR
+    if [[ "$OVN_BUILD_FROM_SOURCE" == "False" ]]; then
+        mkdir_cmd="sudo ${mkdir_cmd}"
+    fi
+
+    $mkdir_cmd
    mkdir -p $OVS_DATADIR

    rm -f $OVS_DATADIR/*.db
    rm -f $OVS_DATADIR/.*.db.~lock~
-    rm -f $OVN_DATADIR/*.db
-    rm -f $OVN_DATADIR/.*.db.~lock~
+    sudo rm -f $OVN_DATADIR/*.db
+    sudo rm -f $OVN_DATADIR/.*.db.~lock~
+    sudo rm -f $OVN_RUNDIR/*.sock
 }

 function _start_ovs {
@@ -595,12 +653,12 @@ function _start_ovs {
                dbcmd+=" --remote=db:hardware_vtep,Global,managers $OVS_DATADIR/vtep.db"
            fi
            dbcmd+=" $OVS_DATADIR/conf.db"
-            _run_process ovsdb-server "$dbcmd"
+            _run_process ovsdb-server "$dbcmd" "" "$STACK_GROUP" "root" "$OVS_RUNDIR"

            # Note: ovn-controller will create and configure br-int once it is started.
            # So, no need to create it now because nothing depends on that bridge here.
            local ovscmd="$OVS_SBINDIR/ovs-vswitchd --log-file --pidfile --detach"
-            _run_process ovs-vswitchd "$ovscmd" "" "$STACK_GROUP" "root"
+            _run_process ovs-vswitchd "$ovscmd" "" "$STACK_GROUP" "root" "$OVS_RUNDIR"
        else
            _start_process "$OVSDB_SERVER_SERVICE"
            _start_process "$OVS_VSWITCHD_SERVICE"
@@ -620,8 +678,8 @@ function _start_ovs {
        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-remote="$OVN_SB_REMOTE"
        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-bridge="br-int"
        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-type="geneve"
-        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$HOST_IP"
-        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:hostname="$LOCAL_HOSTNAME"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$TUNNEL_IP"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:hostname=$(hostname)
        # Select this chassis to host gateway routers
        if [[ "$ENABLE_CHASSIS_AS_GW" == "True" ]]; then
            sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-cms-options="enable-chassis-as-gw"
@@ -635,11 +693,11 @@ function _start_ovs {
        if is_service_enabled ovn-controller-vtep ; then
            ovn_base_setup_bridge br-v
            vtep-ctl add-ps br-v
-            vtep-ctl set Physical_Switch br-v tunnel_ips=$HOST_IP
+            vtep-ctl set Physical_Switch br-v tunnel_ips=$TUNNEL_IP

            enable_service ovs-vtep
            local vtepcmd="$OVS_SCRIPTDIR/ovs-vtep --log-file --pidfile --detach br-v"
-            _run_process ovs-vtep "$vtepcmd" "" "$STACK_GROUP" "root"
+            _run_process ovs-vtep "$vtepcmd" "" "$STACK_GROUP" "root" "$OVS_RUNDIR"

            vtep-ctl set-manager tcp:$HOST_IP:6640
        fi
@@ -662,9 +720,12 @@ function _start_ovn_services {
    if is_service_enabled ovs-vtep ; then
        _start_process "devstack@ovs-vtep.service"
    fi
-    if is_service_enabled q-ovn-metadata-agent; then
+    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent ; then
        _start_process "devstack@q-ovn-metadata-agent.service"
    fi
+    if is_service_enabled q-ovn-agent neutron-ovn-agent ; then
+        _start_process "devstack@q-ovn-agent.service"
+    fi
 }

 # start_ovn() - Start running processes, including screen
@@ -683,23 +744,26 @@ function start_ovn {
            local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_northd"
            local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_northd"

-            _run_process ovn-northd "$cmd" "$stop_cmd"
+            _run_process ovn-northd "$cmd" "$stop_cmd" "$STACK_GROUP" "root" "$OVN_RUNDIR"
        else
            _start_process "$OVN_NORTHD_SERVICE"
        fi

        # Wait for the service to be ready
-        wait_for_sock_file $OVS_RUNDIR/ovnnb_db.sock
-        wait_for_sock_file $OVS_RUNDIR/ovnsb_db.sock
+        # Check for socket and db files for both OVN NB and SB
+        wait_for_sock_file $OVN_RUNDIR/ovnnb_db.sock
+        wait_for_sock_file $OVN_RUNDIR/ovnsb_db.sock
+        wait_for_db_file $OVN_DATADIR/ovnnb_db.db
+        wait_for_db_file $OVN_DATADIR/ovnsb_db.db

        if is_service_enabled tls-proxy; then
-            sudo ovn-nbctl --db=unix:$OVS_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
-            sudo ovn-sbctl --db=unix:$OVS_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
+            sudo ovn-nbctl --db=unix:$OVN_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
+            sudo ovn-sbctl --db=unix:$OVN_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
        fi
-        sudo ovn-nbctl --db=unix:$OVS_RUNDIR/ovnnb_db.sock set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
-        sudo ovn-sbctl --db=unix:$OVS_RUNDIR/ovnsb_db.sock set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
-        sudo ovs-appctl -t $OVS_RUNDIR/ovnnb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
-        sudo ovs-appctl -t $OVS_RUNDIR/ovnsb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
+        sudo ovn-nbctl --db=unix:$OVN_RUNDIR/ovnnb_db.sock set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
+        sudo ovn-sbctl --db=unix:$OVN_RUNDIR/ovnsb_db.sock set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
+        sudo ovs-appctl -t $OVN_RUNDIR/ovnnb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
+        sudo ovs-appctl -t $OVN_RUNDIR/ovnsb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
    fi

    if is_service_enabled ovn-controller ; then
@@ -707,7 +771,7 @@ function start_ovn {
            local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_controller"
            local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_controller"

-            _run_process ovn-controller "$cmd" "$stop_cmd" "$STACK_GROUP" "root"
+            _run_process ovn-controller "$cmd" "$stop_cmd" "$STACK_GROUP" "root" "$OVN_RUNDIR"
        else
            _start_process "$OVN_CONTROLLER_SERVICE"
        fi
@@ -716,18 +780,24 @@ function start_ovn {
    if is_service_enabled ovn-controller-vtep ; then
        if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
            local cmd="$OVS_BINDIR/ovn-controller-vtep --log-file --pidfile --detach --ovnsb-db=$OVN_SB_REMOTE"
-            _run_process ovn-controller-vtep "$cmd" "" "$STACK_GROUP" "root"
+            _run_process ovn-controller-vtep "$cmd" "" "$STACK_GROUP" "root" "$OVN_RUNDIR"
        else
            _start_process "$OVN_CONTROLLER_VTEP_SERVICE"
        fi
    fi

-    if is_service_enabled q-ovn-metadata-agent; then
+    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
        run_process q-ovn-metadata-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_METADATA_BINARY --config-file $OVN_META_CONF"
        # Format logging
        setup_logging $OVN_META_CONF
    fi

+    if is_service_enabled q-ovn-agent neutron-ovn-agent; then
+        run_process q-ovn-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_AGENT_BINARY --config-file $OVN_AGENT_CONF"
+        # Format logging
+        setup_logging $OVN_AGENT_CONF
+    fi
+
    _start_ovn_services
 }

@@ -746,10 +816,18 @@ function _stop_process {
 }

 function stop_ovn {
-    if is_service_enabled q-ovn-metadata-agent; then
-        sudo pkill -9 -f haproxy || :
+    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
+        # pkill takes care not to kill itself, but it may kill its parent
+        # sudo unless we use the "ps | grep [f]oo" trick
+        sudo pkill -9 -f "[h]aproxy" || :
        _stop_process "devstack@q-ovn-metadata-agent.service"
    fi
+    if is_service_enabled q-ovn-agent neutron-ovn-agent; then
+        # pkill takes care not to kill itself, but it may kill its parent
+        # sudo unless we use the "ps | grep [f]oo" trick
+        sudo pkill -9 -f "[h]aproxy" || :
+        _stop_process "devstack@q-ovn-agent.service"
+    fi
    if is_service_enabled ovn-controller-vtep ; then
        _stop_process "$OVN_CONTROLLER_VTEP_SERVICE"
    fi
@@ -792,5 +870,5 @@ function cleanup_ovn {
        _cleanup $ovs_path
    fi

-    sudo rm -f $OVN_RUNDIR
+    sudo rm -rf $OVN_RUNDIR
 }
@@ -7,6 +7,12 @@
 _XTRACE_NEUTRON_OVS_BASE=$(set +o | grep xtrace)
 set +o xtrace

+# Load devstack ovs compliation and loading functions
+source ${TOP_DIR}/lib/neutron_plugins/ovs_source
+
+# Defaults
+# --------
+
 OVS_BRIDGE=${OVS_BRIDGE:-br-int}
 # OVS recognize default 'system' datapath or 'netdev' for userspace datapath
 OVS_DATAPATH_TYPE=${OVS_DATAPATH_TYPE:-system}
@@ -60,26 +66,20 @@ function _neutron_ovs_base_install_ubuntu_dkms {
 }

 function _neutron_ovs_base_install_agent_packages {
-    # Install deps
-    install_package $(get_packages "openvswitch")
-    if is_ubuntu; then
-        _neutron_ovs_base_install_ubuntu_dkms
-        restart_service openvswitch-switch
-    elif is_fedora; then
-        restart_service openvswitch
-        sudo systemctl enable openvswitch
-    elif is_suse; then
-        if [[ $DISTRO == "sle12" ]] && vercmp "$os_RELEASE" "<" "12.2" ; then
+    if [ "$Q_BUILD_OVS_FROM_GIT" == "True" ]; then
+        remove_ovs_packages
+        compile_ovs False /usr/local /var
+        load_conntrack_gre_module
+        start_new_ovs
+    else
+        # Install deps
+        install_package $(get_packages "openvswitch")
+        if is_ubuntu; then
+            _neutron_ovs_base_install_ubuntu_dkms
            restart_service openvswitch-switch
-        else
-            # workaround for https://bugzilla.suse.com/show_bug.cgi?id=1085971
-            if [[ $DISTRO =~ "tumbleweed" ]]; then
-                sudo sed -i -e "s,^OVS_USER_ID=.*,OVS_USER_ID='root:root'," /etc/sysconfig/openvswitch
-            fi
-            restart_service openvswitch || {
-                journalctl -xe || :
-                systemctl status openvswitch
-            }
+        elif is_fedora; then
+            restart_service openvswitch
+            sudo systemctl enable openvswitch
        fi
    fi
 }
@@ -14,6 +14,7 @@

 # Defaults
 # --------
+Q_BUILD_OVS_FROM_GIT=$(trueorfalse False Q_BUILD_OVS_FROM_GIT)

 # Set variables for building OVS from source
 OVS_REPO=${OVS_REPO:-https://github.com/openvswitch/ovs.git}
@@ -32,9 +33,9 @@ function load_module {
    local fatal=$2

    if [ "$(trueorfalse True fatal)" == "True" ]; then
-        sudo modprobe $module || (dmesg && die $LINENO "FAILED TO LOAD $module")
+        sudo modprobe $module || (sudo dmesg && die $LINENO "FAILED TO LOAD $module")
    else
-        sudo modprobe $module || (echo "FAILED TO LOAD $module" && dmesg)
+        sudo modprobe $module || (echo "FAILED TO LOAD $module" && sudo dmesg)
    fi
 }

@@ -86,9 +87,15 @@ function prepare_for_ovs_compilation {

        install_package kernel-devel-$KERNEL_VERSION
        install_package kernel-headers-$KERNEL_VERSION
+        if is_service_enabled tls-proxy; then
+            install_package openssl-devel
+        fi

    elif is_ubuntu ; then
        install_package linux-headers-$KERNEL_VERSION
+        if is_service_enabled tls-proxy; then
+            install_package libssl-dev
+        fi
    fi
 }

@@ -96,7 +103,7 @@ function prepare_for_ovs_compilation {
 function load_ovs_kernel_modules {
    load_module openvswitch
    load_module vport-geneve False
-    dmesg | tail
+    sudo dmesg | tail
 }

 # reload_ovs_kernel_modules() - reload openvswitch kernel module
@@ -157,10 +164,8 @@ function compile_ovs {
    sudo make install
    if [[ "$build_modules" == "True" ]]; then
        sudo make INSTALL_MOD_DIR=kernel/net/openvswitch modules_install
-        reload_ovs_kernel_modules
-    else
-        load_ovs_kernel_modules
    fi
+    reload_ovs_kernel_modules

    cd $_pwd
 }
@@ -175,24 +180,18 @@ function action_openvswitch {
        ${action}_service openvswitch-switch
    elif is_fedora; then
        ${action}_service openvswitch
-    elif is_suse; then
-        if [[ $DISTRO == "sle12" ]] && [[ $os_RELEASE -lt 12.2 ]]; then
-            ${action}_service openvswitch-switch
-        else
-            ${action}_service openvswitch
-        fi
    fi
 }

 # start_new_ovs() - removes old ovs database, creates a new one and starts ovs
 function start_new_ovs {
    sudo rm -f /etc/openvswitch/conf.db /etc/openvswitch/.conf.db~lock~
-    sudo /usr/share/openvswitch/scripts/ovs-ctl start
+    sudo /usr/local/share/openvswitch/scripts/ovs-ctl start
 }

 # stop_new_ovs() - stops ovs
 function stop_new_ovs {
-    local ovs_ctl='/usr/share/openvswitch/scripts/ovs-ctl'
+    local ovs_ctl='/usr/local/share/openvswitch/scripts/ovs-ctl'

    if [ -x $ovs_ctl ] ; then
        sudo $ovs_ctl stop
@@ -211,5 +210,5 @@ function remove_ovs_packages {

 # load_conntrack_gre_module() - loads nf_conntrack_proto_gre kernel module
 function load_conntrack_gre_module {
-    sudo modprobe nf_conntrack_proto_gre
+    load_module nf_conntrack_proto_gre False
 }
@@ -47,7 +47,8 @@ Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True}
 # used for the network.  In case of ofagent, you should add the
 # corresponding entry to your OFAGENT_PHYSICAL_INTERFACE_MAPPINGS.
 # For openvswitch agent, you should add the corresponding entry to
-# your OVS_BRIDGE_MAPPINGS.
+# your OVS_BRIDGE_MAPPINGS and for OVN add the corresponding entry
+# to your OVN_BRIDGE_MAPPINGS.
 #
 # eg.  (ofagent)
 #    Q_USE_PROVIDERNET_FOR_PUBLIC=True
@@ -60,6 +61,11 @@ Q_L3_ROUTER_PER_TENANT=${Q_L3_ROUTER_PER_TENANT:-True}
 #    PUBLIC_PHYSICAL_NETWORK=public
 #    OVS_BRIDGE_MAPPINGS=public:br-ex
 #
+# eg.  (ovn agent)
+#    Q_USER_PROVIDERNET_FOR_PUBLIC=True
+#    PUBLIC_PHYSICAL_NETWORK=public
+#    OVN_BRIDGE_MAPPINGS=public:br-ex
+#
 # The provider-network-type defaults to flat, however, the values
 # PUBLIC_PROVIDERNET_TYPE and PUBLIC_PROVIDERNET_SEGMENTATION_ID could
 # be set to specify the parameters for an alternate network type.
@@ -147,10 +153,6 @@ function _neutron_get_ext_gw_interface {
 }

 function create_neutron_initial_network {
-    local project_id
-    project_id=$(openstack project list | grep " demo " | get_field 1)
-    die_if_not_set $LINENO project_id "Failure retrieving project_id for demo"
-
    # Allow drivers that need to create an initial network to do so here
    if type -p neutron_plugin_create_initial_network_profile > /dev/null; then
        neutron_plugin_create_initial_network_profile $PHYSICAL_NETWORK
@@ -170,15 +172,15 @@ function create_neutron_initial_network {
    if is_provider_network; then
        die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specify the PROVIDER_NETWORK_TYPE"
-        NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create $PHYSICAL_NETWORK --project $project_id --provider-network-type $PROVIDER_NETWORK_TYPE --provider-physical-network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider-segment $SEGMENTATION_ID} --share | grep ' id ' | get_field 2)
-        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $project_id"
+        NET_ID=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" network create $PHYSICAL_NETWORK --provider-network-type $PROVIDER_NETWORK_TYPE --provider-physical-network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider-segment $SEGMENTATION_ID} --share -f value -c id)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            if [ -z $SUBNETPOOL_V4_ID ]; then
                fixed_range_v4=$FIXED_RANGE
            fi
-            SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $project_id"
+            SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} -f value -c id)
+            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME"
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
@@ -187,8 +189,8 @@ function create_neutron_initial_network {
            if [ -z $SUBNETPOOL_V6_ID ]; then
                fixed_range_v6=$IPV6_PROVIDER_FIXED_RANGE
            fi
-            IPV6_SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME $project_id"
+            IPV6_SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} -f value -c id)
+            die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME"
        fi

        if [[ $Q_AGENT == "openvswitch" ]]; then
@@ -197,17 +199,17 @@ function create_neutron_initial_network {
            sudo ip link set $PUBLIC_INTERFACE up
        fi
    else
-        NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create --project $project_id "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
-        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME $project_id"
+        NET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" network create "$PRIVATE_NETWORK_NAME" -f value -c id)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Create IPv4 private subnet
-            SUBNET_ID=$(_neutron_create_private_subnet_v4 $project_id)
+            SUBNET_ID=$(_neutron_create_private_subnet_v4)
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Create IPv6 private subnet
-            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6 $project_id)
+            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6)
        fi
    fi

@@ -215,12 +217,12 @@ function create_neutron_initial_network {
        # Create a router, and add the private subnet as one of its interfaces
        if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
            # create a tenant-owned router.
-            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create --project $project_id $Q_ROUTER_NAME | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $project_id $Q_ROUTER_NAME"
+            ROUTER_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME"
        else
            # Plugin only supports creating a single router, which should be admin owned.
-            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $Q_ROUTER_NAME"
+            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME"
        fi

        EXTERNAL_NETWORK_FLAGS="--external"
@@ -229,9 +231,9 @@ function create_neutron_initial_network {
        fi
        # Create an external network, and a subnet. Configure the external network as router gw
        if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
-            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS --provider-network-type ${PUBLIC_PROVIDERNET_TYPE:-flat} ${PUBLIC_PROVIDERNET_SEGMENTATION_ID:+--provider-segment $PUBLIC_PROVIDERNET_SEGMENTATION_ID} --provider-physical-network ${PUBLIC_PHYSICAL_NETWORK} | grep ' id ' | get_field 2)
+            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS --provider-network-type ${PUBLIC_PROVIDERNET_TYPE:-flat} ${PUBLIC_PROVIDERNET_SEGMENTATION_ID:+--provider-segment $PUBLIC_PROVIDERNET_SEGMENTATION_ID} --provider-physical-network ${PUBLIC_PHYSICAL_NETWORK} -f value -c id)
        else
-            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS | grep ' id ' | get_field 2)
+            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS -f value -c id)
        fi
        die_if_not_set $LINENO EXT_NET_ID "Failure creating EXT_NET_ID for $PUBLIC_NETWORK_NAME"

@@ -249,35 +251,32 @@ function create_neutron_initial_network {

 # Create private IPv4 subnet
 function _neutron_create_private_subnet_v4 {
-    local project_id=$1
    if [ -z $SUBNETPOOL_V4_ID ]; then
        fixed_range_v4=$FIXED_RANGE
    fi
-    local subnet_params="--project $project_id "
-    subnet_params+="--ip-version 4 "
+    local subnet_params="--ip-version 4 "
    if [[ -n "$NETWORK_GATEWAY" ]]; then
        subnet_params+="--gateway $NETWORK_GATEWAY "
    fi
+
    subnet_params+="${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} "
    subnet_params+="${fixed_range_v4:+--subnet-range $fixed_range_v4} "
    subnet_params+="--network $NET_ID $PRIVATE_SUBNET_NAME"
    local subnet_id
-    subnet_id=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create $subnet_params | grep ' id ' | get_field 2)
-    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet for $project_id"
+    subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params -f value -c id)
+    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet"
    echo $subnet_id
 }

 # Create private IPv6 subnet
 function _neutron_create_private_subnet_v6 {
-    local project_id=$1
    die_if_not_set $LINENO IPV6_RA_MODE "IPV6 RA Mode not set"
    die_if_not_set $LINENO IPV6_ADDRESS_MODE "IPV6 Address Mode not set"
    local ipv6_modes="--ipv6-ra-mode $IPV6_RA_MODE --ipv6-address-mode $IPV6_ADDRESS_MODE"
    if [ -z $SUBNETPOOL_V6_ID ]; then
        fixed_range_v6=$FIXED_RANGE_V6
    fi
-    local subnet_params="--project $project_id "
-    subnet_params+="--ip-version 6 "
+    local subnet_params="--ip-version 6 "
    if [[ -n "$IPV6_PRIVATE_NETWORK_GATEWAY" ]]; then
        subnet_params+="--gateway $IPV6_PRIVATE_NETWORK_GATEWAY "
    fi
@@ -285,8 +284,8 @@ function _neutron_create_private_subnet_v6 {
    subnet_params+="${fixed_range_v6:+--subnet-range $fixed_range_v6} "
    subnet_params+="$ipv6_modes --network $NET_ID $IPV6_PRIVATE_SUBNET_NAME "
    local ipv6_subnet_id
-    ipv6_subnet_id=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create $subnet_params | grep ' id ' | get_field 2)
-    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet for $project_id"
+    ipv6_subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params -f value -c id)
+    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet"
    echo $ipv6_subnet_id
 }

@@ -319,7 +318,7 @@ function _neutron_create_public_subnet_v6 {

 # Configure neutron router for IPv4 public access
 function _neutron_configure_router_v4 {
-    openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router add subnet $ROUTER_ID $SUBNET_ID
+    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router add subnet $ROUTER_ID $SUBNET_ID
    # Create a public subnet on the external network
    local id_and_ext_gw_ip
    id_and_ext_gw_ip=$(_neutron_create_public_subnet_v4 $EXT_NET_ID)
@@ -327,10 +326,10 @@ function _neutron_configure_router_v4 {
    ext_gw_ip=$(echo $id_and_ext_gw_ip  | get_field 2)
    PUB_SUBNET_ID=$(echo $id_and_ext_gw_ip | get_field 5)
    # Configure the external network as the default router gateway
-    openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID
+    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID

    # This logic is specific to using OVN or the l3-agent for layer 3
-    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-server) || is_service_enabled q-l3 neutron-l3; then
+    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-api) || is_service_enabled q-l3 neutron-l3; then
        # Configure and enable public bridge
        local ext_gw_interface="none"
        if is_neutron_ovs_base_plugin; then
@@ -363,7 +362,7 @@ function _neutron_configure_router_v4 {

 # Configure neutron router for IPv6 public access
 function _neutron_configure_router_v6 {
-    openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router add subnet $ROUTER_ID $IPV6_SUBNET_ID
+    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router add subnet $ROUTER_ID $IPV6_SUBNET_ID
    # Create a public subnet on the external network
    local ipv6_id_and_ext_gw_ip
    ipv6_id_and_ext_gw_ip=$(_neutron_create_public_subnet_v6 $EXT_NET_ID)
@@ -375,11 +374,11 @@ function _neutron_configure_router_v6 {
    # If the external network has not already been set as the default router
    # gateway when configuring an IPv4 public subnet, do so now
    if [[ "$IP_VERSION" == "6" ]]; then
-        openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID
+        openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID
    fi

    # This logic is specific to using OVN or the l3-agent for layer 3
-    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-server) || is_service_enabled q-l3 neutron-l3; then
+    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-api) || is_service_enabled q-l3 neutron-l3; then
        # if the Linux host considers itself to be a router then it will
        # ignore all router advertisements
        # Ensure IPv6 RAs are accepted on interfaces with a default route.
@@ -396,7 +395,13 @@ function _neutron_configure_router_v6 {
        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        # Configure and enable public bridge
        # Override global IPV6_ROUTER_GW_IP with the true value from neutron
-        IPV6_ROUTER_GW_IP=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" port list -c 'Fixed IP Addresses' | grep $ipv6_pub_subnet_id | awk -F'ip_address' '{ print $2 }' | cut -f2 -d\' | tr '\n' ' ')
+        # NOTE(slaweq): when enforce scopes is enabled in Neutron, router's
+        # gateway ports aren't visible in API because such ports don't belongs
+        # to any tenant. Because of that, at least temporary we need to find
+        # IPv6 address of the router's gateway in a bit different way.
+        # It can be reverted when bug
+        # https://bugs.launchpad.net/neutron/+bug/1959332 will be fixed
+        IPV6_ROUTER_GW_IP=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router show $ROUTER_ID -c external_gateway_info -f json | grep -C 1 $ipv6_pub_subnet_id | grep ip_address | awk '{print $2}' | tr -d '"')
        die_if_not_set $LINENO IPV6_ROUTER_GW_IP "Failure retrieving IPV6_ROUTER_GW_IP"

        if is_neutron_ovs_base_plugin; then
@@ -404,7 +409,10 @@ function _neutron_configure_router_v6 {
            ext_gw_interface=$(_neutron_get_ext_gw_interface)
            local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}

-            # Configure interface for public bridge
+            # Configure interface for public bridge by setting the interface
+            # to "up" in case the job is running entirely private network based
+            # testing.
+            sudo ip link set $ext_gw_interface up
            sudo ip -6 addr replace $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
            # Any IPv6 private subnet that uses the default IPV6 subnet pool
            # and that is plugged into the default router (Q_ROUTER_NAME) will
@@ -427,3 +435,12 @@ function is_networking_extension_supported {
    EXT_LIST=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" extension list --network -c Alias -f value)
    [[ $EXT_LIST =~ $extension ]] && return 0
 }
+
+function plugin_agent_add_l3_agent_extension {
+    local l3_agent_extension=$1
+    if [[ -z "$L3_AGENT_EXTENSIONS" ]]; then
+        L3_AGENT_EXTENSIONS=$l3_agent_extension
+    elif [[ ! ,${L3_AGENT_EXTENSIONS}, =~ ,${l3_agent_extension}, ]]; then
+        L3_AGENT_EXTENSIONS+=",$l3_agent_extension"
+    fi
+}
@@ -12,7 +12,7 @@ AGENT_METERING_BINARY="$NEUTRON_BIN_DIR/neutron-metering-agent"
 METERING_PLUGIN="neutron.services.metering.metering_plugin.MeteringPlugin"

 function neutron_agent_metering_configure_common {
-    _neutron_service_plugin_class_add $METERING_PLUGIN
+    neutron_service_plugin_class_add $METERING_PLUGIN
 }

 function neutron_agent_metering_configure_agent {
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+function configure_placement_service_plugin {
+    neutron_service_plugin_class_add "placement"
+}
+
+function configure_placement_neutron {
+    iniset $NEUTRON_CONF placement auth_type "$NEUTRON_PLACEMENT_AUTH_TYPE"
+    iniset $NEUTRON_CONF placement auth_url "$KEYSTONE_SERVICE_URI"
+    iniset $NEUTRON_CONF placement username "$NEUTRON_PLACEMENT_USERNAME"
+    iniset $NEUTRON_CONF placement password "$SERVICE_PASSWORD"
+    iniset $NEUTRON_CONF placement user_domain_name "$SERVICE_DOMAIN_NAME"
+    iniset $NEUTRON_CONF placement project_name "$SERVICE_TENANT_NAME"
+    iniset $NEUTRON_CONF placement project_domain_name "$SERVICE_DOMAIN_NAME"
+    iniset $NEUTRON_CONF placement region_name "$REGION_NAME"
+}
+
+function configure_placement_extension {
+    configure_placement_service_plugin
+    configure_placement_neutron
+}
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+function configure_qos_service_plugin {
+    neutron_service_plugin_class_add "qos"
+}
+
+
+function configure_qos_core_plugin {
+    configure_qos_$Q_PLUGIN
+}
+
+
+function configure_qos_l2_agent {
+    plugin_agent_add_l2_agent_extension "qos"
+}
+
+
+function configure_qos {
+    configure_qos_service_plugin
+    configure_qos_core_plugin
+    configure_qos_l2_agent
+}
+
+function configure_l3_agent_extension_fip_qos {
+    plugin_agent_add_l3_agent_extension "fip_qos"
+}
+
+function configure_l3_agent_extension_gateway_ip_qos {
+    plugin_agent_add_l3_agent_extension "gateway_ip_qos"
+}
--- a/Show More
+++ b/Show More