[stable-only] Update system locale before PostgreSQL initialization

In Fedora/CentOS9, the PostgreSQL initialization fails with the following message: initdb: invalid locale settings; check LANG and LC_* environment variables This patch installs the required packages and configure the system locale before calling the initialization process. Closes-Bug: #2083482 Change-Id: I2c83e896819b20cd7a1ee8d8ee33354fb047a6d9 (cherry picked from commit e42e6f31a6)
Handle_tags and branches for unmaintained also
2024-10-03 14:54:34 +00:00 · 2024-09-12 12:05:05 +00:00 · 2024-08-06 16:05:47 +05:30 · 2024-07-24 15:49:22 +00:00 · 2024-06-21 18:35:12 +05:30 · 2024-06-12 13:39:58 +00:00
165 changed files with 5504 additions and 2450 deletions
@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/devstack.git
+defaultbranch=unmaintained/zed
@@ -8,6 +8,26 @@
        nodes:
          - controller

+- nodeset:
+    name: openstack-single-node-jammy
+    nodes:
+      - name: controller
+        label: ubuntu-jammy
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: openstack-single-node-focal
+    nodes:
+      - name: controller
+        label: ubuntu-focal
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
 - nodeset:
    name: openstack-single-node-bionic
    nodes:
@@ -29,10 +49,10 @@
          - controller

 - nodeset:
-    name: devstack-single-node-centos-7
+    name: devstack-single-node-centos-9-stream
    nodes:
      - name: controller
-        label: centos-7
+        label: centos-9-stream
    groups:
      - name: tempest
        nodes:
@@ -49,10 +69,10 @@
          - controller

 - nodeset:
-    name: devstack-single-node-fedora-latest
+    name: devstack-single-node-debian-bullseye
    nodes:
      - name: controller
-        label: fedora-29
+        label: debian-bullseye
    groups:
      - name: tempest
        nodes:
@@ -88,6 +108,66 @@
        nodes:
          - compute1

+- nodeset:
+    name: openstack-two-node-centos-9-stream
+    nodes:
+      - name: controller
+        label: centos-9-stream
+      - name: compute1
+        label: centos-9-stream
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
+- nodeset:
+    name: openstack-two-node-focal
+    nodes:
+      - name: controller
+        label: ubuntu-focal
+      - name: compute1
+        label: ubuntu-focal
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
 - nodeset:
    name: openstack-two-node-bionic
    nodes:
@@ -148,6 +228,41 @@
        nodes:
          - compute1

+- nodeset:
+    name: openstack-three-node-focal
+    nodes:
+      - name: controller
+        label: ubuntu-focal
+      - name: compute1
+        label: ubuntu-focal
+      - name: compute2
+        label: ubuntu-focal
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+          - compute2
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+          - compute2
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+          - compute2
+
 - nodeset:
    name: openstack-three-node-bionic
    nodes:
@@ -200,7 +315,6 @@
    required-projects:
      - opendev.org/openstack/devstack
    roles:
-      - zuul: opendev.org/openstack/devstack-gate
      - zuul: opendev.org/openstack/openstack-zuul-jobs
    vars:
      devstack_localrc:
@@ -219,8 +333,8 @@
        VERBOSE_NO_TIMESTAMP: true
        NOVNC_FROM_PACKAGE: true
        ERROR_ON_CLONE: true
-        # Gate jobs can't deal with nested virt. Disable it.
-        LIBVIRT_TYPE: qemu
+        # Gate jobs can't deal with nested virt. Disable it by default.
+        LIBVIRT_TYPE: '{{ devstack_libvirt_type | default("qemu") }}'
      devstack_services:
        # Ignore any default set by devstack. Emit a "disable_all_services".
        base: false
@@ -234,8 +348,10 @@
        '{{ devstack_log_dir }}/devstacklog.txt.summary': logs
        '{{ devstack_log_dir }}/tcpdump.pcap': logs
        '{{ devstack_log_dir }}/worlddump-latest.txt': logs
+        '{{ devstack_log_dir }}/qemu.coredump': logs
        '{{ devstack_full_log}}': logs
        '{{ stage_dir }}/verify_tempest_conf.log': logs
+        '{{ stage_dir }}/performance.json': logs
        '{{ stage_dir }}/apache': logs
        '{{ stage_dir }}/apache_config': logs
        '{{ stage_dir }}/etc': logs
@@ -243,6 +359,7 @@
        /var/log/postgresql: logs
        /var/log/mysql: logs
        /var/log/libvirt: logs
+        /etc/libvirt: logs
        /etc/sudoers: logs
        /etc/sudoers.d: logs
        '{{ stage_dir }}/iptables.txt': logs
@@ -253,8 +370,10 @@
        '{{ stage_dir }}/rpm-qa.txt': logs
        '{{ stage_dir }}/core': logs
        '{{ stage_dir }}/listen53.txt': logs
+        '{{ stage_dir }}/services.txt': logs
        '{{ stage_dir }}/deprecations.log': logs
        '{{ stage_dir }}/audit.log': logs
+        /etc/ceph: logs
        /var/log/ceph: logs
        /var/log/openvswitch: logs
        /var/log/glusterfs: logs
@@ -306,7 +425,7 @@
    description: |
      Minimal devstack base job, intended for use by jobs that need
      less than the normal minimum set of required-projects.
-    nodeset: openstack-single-node-bionic
+    nodeset: openstack-single-node-focal
    required-projects:
      - opendev.org/openstack/requirements
    vars:
@@ -317,17 +436,17 @@
        PUBLIC_BRIDGE_MTU: '{{ external_bridge_mtu }}'
      devstack_services:
        # Shared services
-        dstat: true
+        dstat: false
        etcd3: true
+        memory_tracker: true
        mysql: true
-        peakmem_tracker: true
        rabbit: true
    group-vars:
      subnode:
        devstack_services:
          # Shared services
-          dstat: true
-          peakmem_tracker: true
+          dstat: false
+          memory_tracker: true
        devstack_localrc:
          # Multinode specific settings
          HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}"
@@ -382,6 +501,7 @@
        SWIFT_HASH: 1234123412341234
        DEBUG_LIBVIRT_COREDUMPS: true
        NOVA_VNC_ENABLED: true
+        OVN_DBS_LOG_LEVEL: dbg
      devstack_local_conf:
        post-config:
          $NEUTRON_CONF:
@@ -391,17 +511,16 @@
        # Core services enabled for this branch.
        # This list replaces the test-matrix.
        # Shared services
-        dstat: true
+        dstat: false
        etcd3: true
+        memory_tracker: true
        mysql: true
-        peakmem_tracker: true
        rabbit: true
        tls-proxy: true
        # Keystone services
        key: true
        # Glance services
        g-api: true
-        g-reg: true
        # Nova services
        n-api: true
        n-api-meta: true
@@ -409,22 +528,16 @@
        n-cpu: true
        n-novnc: true
        n-sch: true
+        # Placement service
        placement-api: true
+        # OVN services
+        ovn-controller: true
+        ovn-northd: true
+        ovs-vswitchd: true
+        ovsdb-server: true
        # Neutron services
-        # We need to keep using the neutron-legacy based services for
-        # now until all issues with the new lib/neutron code are solved
-        q-agt: true
-        q-dhcp: true
-        q-l3: true
-        q-meta: true
-        q-metering: true
        q-svc: true
-        # neutron-api: true
-        # neutron-agent: true
-        # neutron-dhcp: true
-        # neutron-l3: true
-        # neutron-metadata-agent: true
-        # neutron-metering: true
+        q-ovn-metadata-agent: true
        # Swift services
        s-account: true
        s-container: true
@@ -435,7 +548,6 @@
        c-bak: true
        c-sch: true
        c-vol: true
-        cinder: true
        # Services we don't need.
        # This section is not really needed, it's for readability.
        horizon: false
@@ -449,17 +561,19 @@
          # Core services enabled for this branch.
          # This list replaces the test-matrix.
          # Shared services
-          dstat: true
-          peakmem_tracker: true
+          dstat: false
+          memory_tracker: true
          tls-proxy: true
          # Nova services
          n-cpu: true
+          # Placement services
          placement-client: true
+          # OVN services
+          ovn-controller: true
+          ovs-vswitchd: true
+          ovsdb-server: true
          # Neutron services
-          # We need to keep using the neutron-legacy based services for
-          # now until all issues with the new lib/neutron code are solved
-          q-agt: true
-          # neutron-agent: true
+          q-ovn-metadata-agent: true
          # Cinder services
          c-bak: true
          c-vol: true
@@ -469,7 +583,7 @@
          # s-*: false
          horizon: false
          tempest: false
-          # Test matrix emits ceilometer but ceilomenter is not installed in the
+          # Test matrix emits ceilometer but ceilometer is not installed in the
          # integrated gate, so specifying the services has not effect.
          # ceilometer-*: false
        devstack_localrc:
@@ -477,76 +591,140 @@
          GLANCE_HOSTPORT: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}:9292"
          Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
          NOVA_VNC_ENABLED: true
+          ENABLE_CHASSIS_AS_GW: false

 - job:
    name: devstack-ipv6
    parent: devstack
    description: |
-      Devstack single node job for integration gate with IPv6.
+      Devstack single node job for integration gate with IPv6,
+      all services and tunnels using IPv6 addresses.
    vars:
      devstack_localrc:
        SERVICE_IP_VERSION: 6
        SERVICE_HOST: ""
-        # IPv6 and certificates known issue with python2
-        # https://bugs.launchpad.net/devstack/+bug/1794929
-        USE_PYTHON3: true
+        TUNNEL_IP_VERSION: 6

 - job:
-    name: devstack-xenial
+    name: devstack-enforce-scope
    parent: devstack
-    nodeset: openstack-single-node-xenial
    description: |
-      Simple singlenode test to verify functionality on devstack
-      side running on Xenial.
+      This job runs the devstack with scope checks enabled.
+    vars:
+      devstack_localrc:
+        ENFORCE_SCOPE: true

 - job:
    name: devstack-multinode
    parent: devstack
-    nodeset: openstack-two-node-bionic
+    nodeset: openstack-two-node-focal
    description: |
      Simple multinode test to verify multinode functionality on devstack side.
      This is not meant to be used as a parent job.
-
- job:
-    name: devstack-multinode-xenial
-    parent: devstack
-    nodeset: openstack-two-node-xenial
-    description: |
-      Simple multinode test to verify multinode functionality on devstack
-      side running on Xenial.
-      This is not meant to be used as a parent job.
+    vars:
+      devstack_localrc:
+        MYSQL_REDUCE_MEMORY: true

 # NOTE(ianw) Platform tests have traditionally been non-voting because
 # we often have to rush things through devstack to stabilise the gate,
 # and these platforms don't have the round-the-clock support to avoid
 # becoming blockers in that situation.
 - job:
-    name: devstack-platform-centos-7
-    parent: tempest-full
-    description: Centos 7 platform test
-    nodeset: devstack-single-node-centos-7
+    name: devstack-platform-centos-9-stream
+    parent: tempest-full-py3
+    description: CentOS 9 Stream platform test
+    nodeset: devstack-single-node-centos-9-stream
+    timeout: 9000
+    # TODO(kopecmartin) n-v until the following is resolved:
+    # https://bugs.launchpad.net/neutron/+bug/1979047
    voting: false
+    vars:
+      configure_swap_size: 4096

 - job:
-    name: devstack-platform-opensuse-15
+    name: devstack-platform-debian-bullseye
    parent: tempest-full-py3
-    description: openSUSE 15.x platform test
-    nodeset: devstack-single-node-opensuse-15
-    voting: false
+    description: Debian Bullseye platform test
+    nodeset: devstack-single-node-debian-bullseye
+    timeout: 9000
+    vars:
+      configure_swap_size: 4096

 - job:
-    name: devstack-platform-fedora-latest
+    name: devstack-platform-ubuntu-jammy
    parent: tempest-full-py3
-    description: Fedora latest platform test
-    nodeset: devstack-single-node-fedora-latest
-    voting: false
+    description: Ubuntu 22.04 LTS (jammy) platform test
+    nodeset: openstack-single-node-jammy
+    timeout: 9000
+    vars:
+      configure_swap_size: 4096
+      devstack_services:
+        # Horizon doesn't like py310
+        horizon: false

 - job:
-    name: devstack-platform-xenial
-    parent: tempest-full-py3
-    description: Ubuntu Xenial platform test
-    nodeset: openstack-single-node-xenial
+    name: devstack-platform-ubuntu-jammy-ovn-source
+    parent: devstack-platform-ubuntu-jammy
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVN from source)
    voting: false
+    vars:
+      devstack_localrc:
+        OVN_BUILD_FROM_SOURCE: True
+        OVN_BRANCH: "v21.06.0"
+        OVS_BRANCH: "a4b04276ab5934d087669ff2d191a23931335c87"
+        OVS_SYSCONFDIR: "/usr/local/etc/openvswitch"
+
+- job:
+    name: devstack-platform-ubuntu-jammy-ovs
+    parent: tempest-full-py3
+    description: Ubuntu 22.04 LTS (jammy) platform test (OVS)
+    nodeset: openstack-single-node-jammy
+    voting: false
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+      devstack_localrc:
+        Q_AGENT: openvswitch
+        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
+        Q_ML2_TENANT_NETWORK_TYPE: vxlan
+      devstack_services:
+        # Horizon doesn't like py310
+        horizon: false
+        # Disable OVN services
+        ovn-northd: false
+        ovn-controller: false
+        ovs-vswitchd: false
+        ovsdb-server: false
+        # Disable Neutron ML2/OVN services
+        q-ovn-metadata-agent: false
+        # Enable Neutron ML2/OVS services
+        q-agt: true
+        q-dhcp: true
+        q-l3: true
+        q-meta: true
+        q-metering: true
+    group-vars:
+      subnode:
+        devstack_services:
+          # Disable OVN services
+          ovn-controller: false
+          ovs-vswitchd: false
+          ovsdb-server: false
+          # Disable Neutron ML2/OVN services
+          q-ovn-metadata-agent: false
+          # Enable Neutron ML2/OVS services
+          q-agt: true
+
+- job:
+    name: devstack-no-tls-proxy
+    parent: tempest-full-py3
+    description: |
+      Tempest job with tls-proxy off.
+
+      Some gates run devstack like this and it follows different code paths.
+    vars:
+      devstack_services:
+        tls-proxy: false

 - job:
    name: devstack-tox-base
@@ -604,6 +782,7 @@

 - job:
    name: devstack-unit-tests
+    nodeset: ubuntu-focal
    description: |
      Runs unit tests on devstack project.

@@ -613,42 +792,41 @@

 - project:
    templates:
-      - integrated-gate
      - integrated-gate-py3
      - publish-openstack-docs-pti
    check:
      jobs:
        - devstack
-        - devstack-xenial
        - devstack-ipv6
-        - devstack-platform-centos-7
-        - devstack-platform-opensuse-15
-        - devstack-platform-fedora-latest
-        - devstack-platform-xenial
+        - devstack-enforce-scope
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bullseye
+        - devstack-platform-ubuntu-jammy
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
        - devstack-multinode
-        - devstack-multinode-xenial
        - devstack-unit-tests
        - openstack-tox-bashate
-        - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa:
-            voting: false
-        - swift-dsvm-functional:
+        - ironic-tempest-bios-ipmi-direct-tinyipa
+        - swift-dsvm-functional
+        - grenade:
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
+        - neutron-ovs-grenade-multinode:
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
+        - neutron-linuxbridge-tempest:
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
+        - neutron-ovn-tempest-ovs-release:
            voting: false
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-grenade:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - neutron-grenade-multinode:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - neutron-tempest-linuxbridge:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - tempest-multinode-full:
+        - tempest-multinode-full-py3:
            voting: false
            irrelevant-files:
              - ^.*\.rst$
@@ -661,24 +839,35 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
+        - nova-ceph-multistore:
+            voting: false
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
    gate:
      jobs:
        - devstack
-        - devstack-xenial
        - devstack-ipv6
+        # TODO(kopecmartin) n-v until the following is resolved:
+        # https://bugs.launchpad.net/neutron/+bug/1979047
+        # - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bullseye
+        - devstack-platform-ubuntu-jammy
+        - devstack-enforce-scope
        - devstack-multinode
-        - devstack-multinode-xenial
        - devstack-unit-tests
        - openstack-tox-bashate
-        - neutron-grenade-multinode:
+        - neutron-ovs-grenade-multinode:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-tempest-linuxbridge:
+        - neutron-linuxbridge-tempest:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-grenade:
+        - ironic-tempest-bios-ipmi-direct-tinyipa
+        - swift-dsvm-functional
+        - grenade:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@@ -690,6 +879,13 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
+        # TODO(elod.illes): nova-ceph-multistore is broken in this branch,
+        # so it is set as non-voting in check queue and commented out here
+        # in gate queue until the job gets fixed.
+        # - nova-ceph-multistore:
+        #     irrelevant-files:
+        #       - ^.*\.rst$
+        #       - ^doc/.*$
    # Please add a note on each job and conditions for the job not
    # being experimental any more, so we can keep this list somewhat
    # pruned.
@@ -715,27 +911,19 @@
        - neutron-fullstack-with-uwsgi
        - neutron-functional-with-uwsgi
        - neutron-tempest-with-uwsgi
-        - devstack-plugin-ceph-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - devstack-plugin-ceph-tempest-py3:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-tempest-dvr:
+        - neutron-ovs-tempest-dvr:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - legacy-tempest-dsvm-neutron-dvr-multinode-full:
+        - neutron-ovs-tempest-dvr-ha-multinode-full:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-tempest-dvr-ha-multinode-full:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - legacy-tempest-dsvm-lvm-multibackend:
+        - cinder-tempest-lvm-multibackend:
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
@@ -743,3 +931,4 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
+        - devstack-no-tls-proxy
@@ -0,0 +1,19 @@
+The source repository for this project can be found at:
+
+   https://opendev.org/openstack/devstack
+
+Pull requests submitted through GitHub are not monitored.
+
+To start contributing to OpenStack, follow the steps in the contribution guide
+to set up and use Gerrit:
+
+   https://docs.openstack.org/contributors/code-and-documentation/quick-start.html
+
+Bugs should be filed on Launchpad:
+
+   https://bugs.launchpad.net/devstack
+
+For more specific information about contributing to this repository, see the
+Devstack contributor guide:
+
+   https://docs.openstack.org/devstack/latest/contributor/contributing.html
@@ -10,7 +10,7 @@ and so is limited to Bash (version 4 and up) and compatible shells.
 Shell script was chosen because it best illustrates the steps used to
 set up and interact with OpenStack components.

-DevStack's official repository is located on git.openstack.org at
+DevStack's official repository is located on opendev.org at
 https://opendev.org/openstack/devstack.  Besides the master branch that
 tracks the OpenStack trunk branches a separate branch is maintained for all
 OpenStack releases starting with Diablo (stable/diablo).
@@ -23,7 +23,7 @@ __ contribute_
 .. _contribute: https://docs.openstack.org/infra/manual/developers.html

 __ lp_
-.. _lp: https://launchpad.net/~devstack
+.. _lp: https://launchpad.net/devstack

 The `Gerrit review
 queue <https://review.opendev.org/#/q/project:openstack/devstack>`__
@@ -74,8 +74,7 @@ of test of specific fragile functions in the ``functions`` and

 ``tools`` - Contains a collection of stand-alone scripts. While these
 may reference the top-level DevStack configuration they can generally be
-run alone. There are also some sub-directories to support specific
-environments such as XenServer.
+run alone.


 Scripts
@@ -163,7 +162,7 @@ Documentation

 The DevStack repo now contains all of the static pages of devstack.org in
 the ``doc/source`` directory. The OpenStack CI system rebuilds the docs after every
-commit and updates devstack.org (now a redirect to docs.openstack.org/developer/devstack).
+commit and updates devstack.org (now a redirect to https://docs.openstack.org/devstack/latest/).

 All of the scripts are processed with shocco_ to render them with the comments
 as text describing the script below.  For this reason we tend to be a little
@@ -275,9 +274,6 @@ your change
  even years from now -- why we were motivated to make a change at the
  time.

-* **Reviewers** -- please see ``MAINTAINERS.rst`` for a list of people
-  that should be added to reviews of various sub-systems.
-

 Making Changes, Testing, and CI
 -------------------------------
@@ -1,92 +0,0 @@
-MAINTAINERS
-===========
-
-
-Overview
--------
-
-The following is a list of people known to have interests in
-particular areas or sub-systems of devstack.
-
-It is a rather general guide intended to help seed the initial
-reviewers list of a change.  A +1 on a review from someone identified
-as being a maintainer of its affected area is a very positive flag to
-the core team for the veracity of the change.
-
-The ``devstack-core`` group can still be added to all reviews.
-
-
-Format
-~~~~~~
-
-The format of the file is the name of the maintainer and their
-gerrit-registered email.
-
-
-Maintainers
-----------
-
-.. contents:: :local:
-
-
-Ceph
-~~~~
-
-* Sebastien Han <sebastien.han@enovance.com>
-
-Cinder
-~~~~~~
-
-Fedora/CentOS/RHEL
-~~~~~~~~~~~~~~~~~~
-
-* Ian Wienand <iwienand@redhat.com>
-
-Neutron
-~~~~~~~
-
-MidoNet
-~~~~~~~
-
-* Jaume Devesa <devvesa@gmail.com>
-* Ryu Ishimoto <ryu@midokura.com>
-* YAMAMOTO Takashi <yamamoto@midokura.com>
-
-OpenDaylight
-~~~~~~~~~~~~
-
-* Kyle Mestery <mestery@mestery.com>
-
-OpenFlow Agent (ofagent)
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-* YAMAMOTO Takashi <yamamoto@valinux.co.jp>
-* Fumihiko Kakuma <kakuma@valinux.co.jp>
-
-Swift
-~~~~~
-
-* Chmouel Boudjnah <chmouel@enovance.com>
-
-SUSE
-~~~~
-
-* Ralf Haferkamp <rhafer@suse.de>
-* Vincent Untz <vuntz@suse.com>
-
-Tempest
-~~~~~~~
-
-Xen
-~~~
-* Bob Ball <bob.ball@citrix.com>
-
-Zaqar (Marconi)
-~~~~~~~~~~~~~~~
-
-* Flavio Percoco <flaper87@gmail.com>
-* Malini Kamalambal <malini.kamalambal@rackspace.com>
-
-Oracle Linux
-~~~~~~~~~~~~
-* Wiekus Beukes <wiekus.beukes@oracle.com>
@@ -113,7 +113,7 @@ cleanup_rpc_backend
 cleanup_database

 # Clean out data and status
-sudo rm -rf $DATA_DIR $DEST/status
+sudo rm -rf $DATA_DIR $DEST/status $DEST/async

 # Clean out the log file and log directories
 if [[ -n "$LOGFILE" ]] && [[ -f "$LOGFILE" ]]; then
@@ -123,12 +123,10 @@ if [[ -n "$LOGDIR" ]] && [[ -d "$LOGDIR" ]]; then
    sudo rm -rf $LOGDIR
 fi

-# Clean out the systemd user unit files if systemd was used.
-if [[ "$USE_SYSTEMD" = "True" ]]; then
-    sudo find $SYSTEMD_DIR -type f -name '*devstack@*service' -delete
-    # Make systemd aware of the deletion.
-    $SYSTEMCTL daemon-reload
-fi
+# Clean out the systemd unit files.
+sudo find $SYSTEMD_DIR -type f -name '*devstack@*service' -delete
+# Make systemd aware of the deletion.
+$SYSTEMCTL daemon-reload

 # Clean up venvs
 DIRS_TO_CLEAN="$WHEELHOUSE ${PROJECT_VENV[@]} .config/openstack"
@@ -147,12 +145,5 @@ done

 rm -rf ~/.config/openstack

-# Clean up all *.pyc files
-if [[ -n "$DEST" ]] && [[ -d "$DEST" ]]; then
-    find_version=`find --version | awk '{ print $NF; exit}'`
-    if vercmp "$find_version" "<" "4.2.3" ; then
-        sudo find $DEST -name "*.pyc" -print0 | xargs -0 rm
-    else
-        sudo find $DEST -name "*.pyc" -delete
-    fi
-fi
+# Clear any fstab entries made
+sudo sed -i '/.*comment=devstack-.*/ d' /etc/fstab
@@ -2,8 +2,8 @@ pbr>=2.0.0,!=2.1.0

 Pygments
 docutils
-sphinx>=1.6.2
-openstackdocstheme>=1.20.0
+sphinx>=2.0.0,!=2.1.0 # BSD
+openstackdocstheme>=2.2.1 # Apache-2.0
 nwdiag
 blockdiag
 sphinxcontrib-blockdiag
@@ -30,9 +30,15 @@ extensions = [ 'sphinx.ext.autodoc',
               'sphinxcontrib.nwdiag' ]

 # openstackdocstheme options
-repository_name = 'openstack-dev/devstack'
-bug_project = 'devstack'
-bug_tag = ''
+openstackdocs_repo_name = 'openstack-dev/devstack'
+openstackdocs_pdf_link = True
+openstackdocs_bug_project = 'devstack'
+openstackdocs_bug_tag = ''
+openstackdocs_auto_name = False
+# This repo is not tagged, so don't set versions
+openstackdocs_auto_version = False
+version = ''
+release = ''

 todo_include_todos = True

@@ -81,7 +87,7 @@ add_module_names = False
 show_authors = False

 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'

 # A list of ignored prefixes for module index sorting.
 modindex_common_prefix = ['DevStack-doc.']
@@ -279,7 +279,7 @@ number of days of old log files to keep.

 ::

-    LOGDAYS=1
+    LOGDAYS=2

 Some coloring is used during the DevStack runs to make it easier to
 see what is going on. This can be disabled with::
@@ -430,17 +430,6 @@ Python bindings added when they are enabled.

  ADDITIONAL_VENV_PACKAGES="python-foo, python-bar"

-Use python3
------------
-
-By default ``stack.sh`` uses python2 (the exact version set by the
-``PYTHON2_VERSION``). This can be overriden so devstack will run
-python3 (the exact version set by ``PYTHON3_VERSION``).
-
-::
-
-  USE_PYTHON3=True
-
 A clean install every time
 --------------------------

@@ -532,8 +521,8 @@ behavior:
 can be configured with any valid IPv6 prefix. The default values make
 use of an auto-generated ``IPV6_GLOBAL_ID`` to comply with RFC4193.

-Service Version
-~~~~~~~~~~~~~~~
+Service IP Version
+~~~~~~~~~~~~~~~~~~

 DevStack can enable service operation over either IPv4 or IPv6 by
 setting ``SERVICE_IP_VERSION`` to either ``SERVICE_IP_VERSION=4`` or
@@ -553,6 +542,27 @@ optionally be used to alter the default IPv6 address::

  HOST_IPV6=${some_local_ipv6_address}

+Tunnel IP Version
+~~~~~~~~~~~~~~~~~
+
+DevStack can enable tunnel operation over either IPv4 or IPv6 by
+setting ``TUNNEL_IP_VERSION`` to either ``TUNNEL_IP_VERSION=4`` or
+``TUNNEL_IP_VERSION=6`` respectively.
+
+When set to ``4`` Neutron will use an IPv4 address for tunnel endpoints,
+for example, ``HOST_IP``.
+
+When set to ``6`` Neutron will use an IPv6 address for tunnel endpoints,
+for example, ``HOST_IPV6``.
+
+The default value for this setting is ``4``.  Dual-mode support, for
+example ``4+6`` is not supported, as this value must match the address
+family of the local tunnel endpoint IP(v6) address.
+
+The value of ``TUNNEL_IP_VERSION`` has a direct relationship to the
+setting of ``TUNNEL_ENDPOINT_IP``, which will default to ``HOST_IP``
+when set to ``4``, and ``HOST_IPV6`` when set to ``6``.
+
 Multi-node setup
 ~~~~~~~~~~~~~~~~

@@ -639,12 +649,6 @@ outside of tox. If you would like to install it add the following to your
    INSTALL_TEMPEST=True


-Xenserver
-~~~~~~~~~
-
-If you would like to use Xenserver as the hypervisor, please refer to
-the instructions in ``./tools/xen/README.md``.
-
 Cinder
 ~~~~~~

@@ -659,6 +663,12 @@ with ``VOLUME_BACKING_FILE_SIZE``.
  VOLUME_NAME_PREFIX="volume-"
  VOLUME_BACKING_FILE_SIZE=24G

+When running highly concurrent tests, the default per-project quotas
+for volumes, backups, or snapshots may be too small.  These can be
+adjusted by setting ``CINDER_QUOTA_VOLUMES``, ``CINDER_QUOTA_BACKUPS``,
+or ``CINDER_QUOTA_SNAPSHOTS`` to the desired value.  (The default for
+each is 10.)
+

 Keystone
 ~~~~~~~~
@@ -683,7 +693,6 @@ In RegionTwo:

    disable_service horizon
    KEYSTONE_SERVICE_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
-    KEYSTONE_AUTH_HOST=<KEYSTONE_IP_ADDRESS_FROM_REGION_ONE>
    REGION_NAME=RegionTwo
    KEYSTONE_REGION_NAME=RegionOne

@@ -696,15 +705,22 @@ KEYSTONE_REGION_NAME to specify the region of Keystone service.
 KEYSTONE_REGION_NAME has a default value the same as REGION_NAME thus we omit
 it in the configuration of RegionOne.

-Disabling Identity API v2
-+++++++++++++++++++++++++
+Glance
++++++

-The Identity API v2 is deprecated as of Mitaka and it is recommended to only
-use the v3 API. It is possible to setup keystone without v2 API, by doing:
+The default image size quota of 1GiB may be too small if larger images
+are to be used. Change the default at setup time with:

 ::

-    ENABLE_IDENTITY_V2=False
+    GLANCE_LIMIT_IMAGE_SIZE_TOTAL=5000
+
+or at runtime via:
+
+::
+
+    openstack --os-cloud devstack-system-admin registered limit update \
+      --service glance --default-limit 5000 --region RegionOne image_size_total

 .. _arch-configuration:

@@ -0,0 +1,56 @@
+============================
+So You Want to Contribute...
+============================
+
+For general information on contributing to OpenStack, please check out the
+`contributor guide <https://docs.openstack.org/contributors/>`_ to get started.
+It covers all the basics that are common to all OpenStack projects: the accounts
+you need, the basics of interacting with our Gerrit review system, how we
+communicate as a community, etc.
+
+Below will cover the more project specific information you need to get started
+with Devstack.
+
+Communication
+~~~~~~~~~~~~~
+* IRC channel ``#openstack-qa`` at OFTC.
+* Mailing list (prefix subjects with ``[qa][devstack]`` for faster responses)
+  http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-discuss
+
+Contacting the Core Team
+~~~~~~~~~~~~~~~~~~~~~~~~
+Please refer to the `Devstack Core Team
+<https://review.opendev.org/#/admin/groups/50,members>`_ contacts.
+
+New Feature Planning
+~~~~~~~~~~~~~~~~~~~~
+If you want to propose a new feature please read `Feature Proposal Process`_
+Devstack features are tracked on `Launchpad BP <https://blueprints.launchpad.net/devstack>`_.
+
+Task Tracking
+~~~~~~~~~~~~~
+We track our tasks in `Launchpad <https://bugs.launchpad.net/devstack>`_.
+
+Reporting a Bug
+~~~~~~~~~~~~~~~
+You found an issue and want to make sure we are aware of it? You can do so on
+`Launchpad <https://bugs.launchpad.net/devstack/+filebug>`__.
+More info about Launchpad usage can be found on `OpenStack docs page
+<https://docs.openstack.org/contributors/common/task-tracking.html#launchpad>`_
+
+Getting Your Patch Merged
+~~~~~~~~~~~~~~~~~~~~~~~~~
+All changes proposed to the Devstack require two ``Code-Review +2`` votes from
+Devstack core reviewers before one of the core reviewers can approve the patch
+by giving ``Workflow +1`` vote. One exception is for patches to unblock the gate
+which can be approved by single core reviewers.
+
+Project Team Lead Duties
+~~~~~~~~~~~~~~~~~~~~~~~~
+All common PTL duties are enumerated in the `PTL guide
+<https://docs.openstack.org/project-team-guide/ptl.html>`_.
+
+The Release Process for QA is documented in `QA Release Process
+<https://wiki.openstack.org/wiki/QA/releases>`_.
+
+.. _Feature Proposal Process: https://wiki.openstack.org/wiki/QA#Feature_Proposal_.26_Design_discussions
@@ -10,6 +10,7 @@ Walk through various setups used by stackers

 .. toctree::
   :glob:
+   :hidden:
   :maxdepth: 1

   guides/single-vm
@@ -68,6 +69,11 @@ Nova and devstack

 Guide to working with nova features :doc:`Nova and devstack <guides/nova>`.

+Configure Load-Balancer Version 2
+-----------------------------------
+
+Guide on :doc:`Configure Load-Balancer Version 2 <guides/devstack-with-lbaas-v2>`.
+
 Deploying DevStack with LDAP
 ----------------------------

@@ -41,9 +41,6 @@ Edit your ``/opt/stack/devstack/local.conf`` to look like
    # If you are enabling barbican for TLS offload in Octavia, include it here.
    # enable_plugin barbican https://opendev.org/openstack/barbican

-    # If you have python3 available:
-    # USE_PYTHON3=True
-
    # ===== BEGIN localrc =====
    DATABASE_PASSWORD=password
    ADMIN_PASSWORD=password
@@ -62,7 +59,7 @@ Edit your ``/opt/stack/devstack/local.conf`` to look like
    ENABLED_SERVICES+=,n-api,n-crt,n-cpu,n-cond,n-sch,n-api-meta,n-sproxy
    ENABLED_SERVICES+=,placement-api,placement-client
    # Glance
-    ENABLED_SERVICES+=,g-api,g-reg
+    ENABLED_SERVICES+=,g-api
    # Neutron
    ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron
    ENABLED_SERVICES+=,octavia,o-cw,o-hk,o-hm,o-api
@@ -75,13 +75,21 @@ Otherwise create the stack user:

    useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+::
+
+    chmod +x /opt/stack
+
 This user will be making many changes to your system during installation
 and operation so it needs to have sudo privileges to root without a
 password:

 ::

-    echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 From here on use the ``stack`` user. **Logout** and **login** as the
 ``stack`` user.
@@ -169,17 +177,12 @@ machines, create a ``local.conf`` with:
    MYSQL_HOST=$SERVICE_HOST
    RABBIT_HOST=$SERVICE_HOST
    GLANCE_HOSTPORT=$SERVICE_HOST:9292
-    ENABLED_SERVICES=n-cpu,q-agt,n-api-meta,c-vol,placement-client
+    ENABLED_SERVICES=n-cpu,c-vol,placement-client,ovn-controller,ovs-vswitchd,ovsdb-server,q-ovn-metadata-agent
    NOVA_VNC_ENABLED=True
    NOVNCPROXY_URL="http://$SERVICE_HOST:6080/vnc_lite.html"
    VNCSERVER_LISTEN=$HOST_IP
    VNCSERVER_PROXYCLIENT_ADDRESS=$VNCSERVER_LISTEN

-**Note:** the ``n-api-meta`` service is a version of the api server
-that only serves the metadata service. It's needed because the
-computes created won't have a routing path to the metadata service on
-the controller.
-
 Fire up OpenStack:

 ::
@@ -400,7 +403,7 @@ SSH keys need to be exchanged between each compute node:

 3. Verify that login via ssh works without a password::

-        ssh -i /root/.ssh/id_rsa.pub stack@DESTINATION
+        ssh -i /root/.ssh/id_rsa stack@DESTINATION

 In essence, this means that every compute node's root user's public RSA key
 must exist in every other compute node's stack user's authorized_keys file and
@@ -49,13 +49,21 @@ below)

    $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it will need
 to have sudo privileges:

 .. code-block:: console

    $ apt-get install sudo -y || yum install -y sudo
-    $ echo "stack ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    $ echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack

 .. note:: On some systems you may need to use ``sudo visudo``.

@@ -38,10 +38,9 @@ Install Linux

 Start with a clean and minimal install of a Linux system. DevStack
 attempts to support the two latest LTS releases of Ubuntu, the
-latest/current Fedora version, CentOS/RHEL 7, as well as Debian and
-OpenSUSE.
+latest/current Fedora version, CentOS/RHEL 8 and OpenSUSE.

-If you do not have a preference, Ubuntu 18.04 (Bionic Beaver) is the
+If you do not have a preference, Ubuntu 20.04 (Focal Fossa) is the
 most tested, and will probably go the smoothest.

 Add Stack User (optional)
@@ -58,13 +57,21 @@ to run DevStack with

   $ sudo useradd -s /bin/bash -d /opt/stack -m stack

+Ensure home directory for the ``stack`` user has executable permission for all,
+as RHEL based distros create it with ``700`` and Ubuntu 21.04+ with ``750``
+which can cause issues during deployment.
+
+.. code-block:: console
+
+    $ sudo chmod +x /opt/stack
+
 Since this user will be making many changes to your system, it should
 have sudo privileges:

 .. code-block:: console

    $ echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack
-    $ sudo su - stack
+    $ sudo -u stack -i

 Download DevStack
 -----------------
@@ -151,6 +158,13 @@ Get :doc:`the big picture <overview>` of what we are trying to do
 with devstack, and help us by :doc:`contributing to the project
 <hacking>`.

+If you are a new contributor to devstack please refer: :doc:`contributor/contributing`
+
+.. toctree::
+   :hidden:
+
+   contributor/contributing
+
 Contents
 ++++++++

@@ -40,7 +40,7 @@ updates. Tempest tests will work in this environment.
 Locally Accessible Guests
 =========================

-If you want to make you guests accessible from other machines on your
+If you want to make your guests accessible from other machines on your
 network, we have to connect ``br-ex`` to a physical interface.

 Dedicated Guest Interface
@@ -81,7 +81,7 @@ of addresses, and have them all exactly correct.
   [[local|localrc]]
   PUBLIC_INTERFACE=eth0
   HOST_IP=10.42.0.52
-   FLOATING_RANGE=10.42.0.52/24
+   FLOATING_RANGE=10.42.0.0/24
   PUBLIC_NETWORK_GATEWAY=10.42.0.1
   Q_FLOATING_ALLOCATION_POOL=start=10.42.0.250,end=10.42.0.254

@@ -31,16 +31,14 @@ openstack/ceilometer                     `https://opendev.org/openstack/ceilomet
 openstack/ceilometer-powervm             `https://opendev.org/openstack/ceilometer-powervm <https://opendev.org/openstack/ceilometer-powervm>`__
 openstack/cinderlib                      `https://opendev.org/openstack/cinderlib <https://opendev.org/openstack/cinderlib>`__
 openstack/cloudkitty                     `https://opendev.org/openstack/cloudkitty <https://opendev.org/openstack/cloudkitty>`__
-openstack/congress                       `https://opendev.org/openstack/congress <https://opendev.org/openstack/congress>`__
 openstack/cyborg                         `https://opendev.org/openstack/cyborg <https://opendev.org/openstack/cyborg>`__
 openstack/designate                      `https://opendev.org/openstack/designate <https://opendev.org/openstack/designate>`__
 openstack/devstack-plugin-amqp1          `https://opendev.org/openstack/devstack-plugin-amqp1 <https://opendev.org/openstack/devstack-plugin-amqp1>`__
 openstack/devstack-plugin-ceph           `https://opendev.org/openstack/devstack-plugin-ceph <https://opendev.org/openstack/devstack-plugin-ceph>`__
 openstack/devstack-plugin-container      `https://opendev.org/openstack/devstack-plugin-container <https://opendev.org/openstack/devstack-plugin-container>`__
 openstack/devstack-plugin-kafka          `https://opendev.org/openstack/devstack-plugin-kafka <https://opendev.org/openstack/devstack-plugin-kafka>`__
-openstack/devstack-plugin-pika           `https://opendev.org/openstack/devstack-plugin-pika <https://opendev.org/openstack/devstack-plugin-pika>`__
-openstack/devstack-plugin-zmq            `https://opendev.org/openstack/devstack-plugin-zmq <https://opendev.org/openstack/devstack-plugin-zmq>`__
-openstack/dragonflow                     `https://opendev.org/openstack/dragonflow <https://opendev.org/openstack/dragonflow>`__
+openstack/devstack-plugin-nfs            `https://opendev.org/openstack/devstack-plugin-nfs <https://opendev.org/openstack/devstack-plugin-nfs>`__
+openstack/devstack-plugin-open-cas       `https://opendev.org/openstack/devstack-plugin-open-cas <https://opendev.org/openstack/devstack-plugin-open-cas>`__
 openstack/ec2-api                        `https://opendev.org/openstack/ec2-api <https://opendev.org/openstack/ec2-api>`__
 openstack/freezer                        `https://opendev.org/openstack/freezer <https://opendev.org/openstack/freezer>`__
 openstack/freezer-api                    `https://opendev.org/openstack/freezer-api <https://opendev.org/openstack/freezer-api>`__
@@ -50,9 +48,8 @@ openstack/heat                           `https://opendev.org/openstack/heat <ht
 openstack/heat-dashboard                 `https://opendev.org/openstack/heat-dashboard <https://opendev.org/openstack/heat-dashboard>`__
 openstack/ironic                         `https://opendev.org/openstack/ironic <https://opendev.org/openstack/ironic>`__
 openstack/ironic-inspector               `https://opendev.org/openstack/ironic-inspector <https://opendev.org/openstack/ironic-inspector>`__
+openstack/ironic-prometheus-exporter     `https://opendev.org/openstack/ironic-prometheus-exporter <https://opendev.org/openstack/ironic-prometheus-exporter>`__
 openstack/ironic-ui                      `https://opendev.org/openstack/ironic-ui <https://opendev.org/openstack/ironic-ui>`__
-openstack/karbor                         `https://opendev.org/openstack/karbor <https://opendev.org/openstack/karbor>`__
-openstack/karbor-dashboard               `https://opendev.org/openstack/karbor-dashboard <https://opendev.org/openstack/karbor-dashboard>`__
 openstack/keystone                       `https://opendev.org/openstack/keystone <https://opendev.org/openstack/keystone>`__
 openstack/kuryr-kubernetes               `https://opendev.org/openstack/kuryr-kubernetes <https://opendev.org/openstack/kuryr-kubernetes>`__
 openstack/kuryr-libnetwork               `https://opendev.org/openstack/kuryr-libnetwork <https://opendev.org/openstack/kuryr-libnetwork>`__
@@ -64,25 +61,16 @@ openstack/manila-tempest-plugin          `https://opendev.org/openstack/manila-t
 openstack/manila-ui                      `https://opendev.org/openstack/manila-ui <https://opendev.org/openstack/manila-ui>`__
 openstack/masakari                       `https://opendev.org/openstack/masakari <https://opendev.org/openstack/masakari>`__
 openstack/mistral                        `https://opendev.org/openstack/mistral <https://opendev.org/openstack/mistral>`__
-openstack/monasca-analytics              `https://opendev.org/openstack/monasca-analytics <https://opendev.org/openstack/monasca-analytics>`__
 openstack/monasca-api                    `https://opendev.org/openstack/monasca-api <https://opendev.org/openstack/monasca-api>`__
-openstack/monasca-ceilometer             `https://opendev.org/openstack/monasca-ceilometer <https://opendev.org/openstack/monasca-ceilometer>`__
 openstack/monasca-events-api             `https://opendev.org/openstack/monasca-events-api <https://opendev.org/openstack/monasca-events-api>`__
-openstack/monasca-log-api                `https://opendev.org/openstack/monasca-log-api <https://opendev.org/openstack/monasca-log-api>`__
 openstack/monasca-tempest-plugin         `https://opendev.org/openstack/monasca-tempest-plugin <https://opendev.org/openstack/monasca-tempest-plugin>`__
-openstack/monasca-transform              `https://opendev.org/openstack/monasca-transform <https://opendev.org/openstack/monasca-transform>`__
 openstack/murano                         `https://opendev.org/openstack/murano <https://opendev.org/openstack/murano>`__
 openstack/networking-bagpipe             `https://opendev.org/openstack/networking-bagpipe <https://opendev.org/openstack/networking-bagpipe>`__
 openstack/networking-baremetal           `https://opendev.org/openstack/networking-baremetal <https://opendev.org/openstack/networking-baremetal>`__
 openstack/networking-bgpvpn              `https://opendev.org/openstack/networking-bgpvpn <https://opendev.org/openstack/networking-bgpvpn>`__
-openstack/networking-calico              `https://opendev.org/openstack/networking-calico <https://opendev.org/openstack/networking-calico>`__
 openstack/networking-generic-switch      `https://opendev.org/openstack/networking-generic-switch <https://opendev.org/openstack/networking-generic-switch>`__
 openstack/networking-hyperv              `https://opendev.org/openstack/networking-hyperv <https://opendev.org/openstack/networking-hyperv>`__
-openstack/networking-l2gw                `https://opendev.org/openstack/networking-l2gw <https://opendev.org/openstack/networking-l2gw>`__
-openstack/networking-midonet             `https://opendev.org/openstack/networking-midonet <https://opendev.org/openstack/networking-midonet>`__
 openstack/networking-odl                 `https://opendev.org/openstack/networking-odl <https://opendev.org/openstack/networking-odl>`__
-openstack/networking-onos                `https://opendev.org/openstack/networking-onos <https://opendev.org/openstack/networking-onos>`__
-openstack/networking-ovn                 `https://opendev.org/openstack/networking-ovn <https://opendev.org/openstack/networking-ovn>`__
 openstack/networking-powervm             `https://opendev.org/openstack/networking-powervm <https://opendev.org/openstack/networking-powervm>`__
 openstack/networking-sfc                 `https://opendev.org/openstack/networking-sfc <https://opendev.org/openstack/networking-sfc>`__
 openstack/neutron                        `https://opendev.org/openstack/neutron <https://opendev.org/openstack/neutron>`__
@@ -95,34 +83,33 @@ openstack/neutron-vpnaas-dashboard       `https://opendev.org/openstack/neutron-
 openstack/nova-powervm                   `https://opendev.org/openstack/nova-powervm <https://opendev.org/openstack/nova-powervm>`__
 openstack/octavia                        `https://opendev.org/openstack/octavia <https://opendev.org/openstack/octavia>`__
 openstack/octavia-dashboard              `https://opendev.org/openstack/octavia-dashboard <https://opendev.org/openstack/octavia-dashboard>`__
+openstack/octavia-tempest-plugin         `https://opendev.org/openstack/octavia-tempest-plugin <https://opendev.org/openstack/octavia-tempest-plugin>`__
 openstack/openstacksdk                   `https://opendev.org/openstack/openstacksdk <https://opendev.org/openstack/openstacksdk>`__
-openstack/os-loganalyze                  `https://opendev.org/openstack/os-loganalyze <https://opendev.org/openstack/os-loganalyze>`__
 openstack/osprofiler                     `https://opendev.org/openstack/osprofiler <https://opendev.org/openstack/osprofiler>`__
 openstack/oswin-tempest-plugin           `https://opendev.org/openstack/oswin-tempest-plugin <https://opendev.org/openstack/oswin-tempest-plugin>`__
-openstack/panko                          `https://opendev.org/openstack/panko <https://opendev.org/openstack/panko>`__
+openstack/ovn-octavia-provider           `https://opendev.org/openstack/ovn-octavia-provider <https://opendev.org/openstack/ovn-octavia-provider>`__
 openstack/patrole                        `https://opendev.org/openstack/patrole <https://opendev.org/openstack/patrole>`__
-openstack/qinling                        `https://opendev.org/openstack/qinling <https://opendev.org/openstack/qinling>`__
-openstack/qinling-dashboard              `https://opendev.org/openstack/qinling-dashboard <https://opendev.org/openstack/qinling-dashboard>`__
-openstack/rally                          `https://opendev.org/openstack/rally <https://opendev.org/openstack/rally>`__
 openstack/rally-openstack                `https://opendev.org/openstack/rally-openstack <https://opendev.org/openstack/rally-openstack>`__
 openstack/sahara                         `https://opendev.org/openstack/sahara <https://opendev.org/openstack/sahara>`__
 openstack/sahara-dashboard               `https://opendev.org/openstack/sahara-dashboard <https://opendev.org/openstack/sahara-dashboard>`__
-openstack/searchlight                    `https://opendev.org/openstack/searchlight <https://opendev.org/openstack/searchlight>`__
-openstack/searchlight-ui                 `https://opendev.org/openstack/searchlight-ui <https://opendev.org/openstack/searchlight-ui>`__
 openstack/senlin                         `https://opendev.org/openstack/senlin <https://opendev.org/openstack/senlin>`__
 openstack/shade                          `https://opendev.org/openstack/shade <https://opendev.org/openstack/shade>`__
+openstack/skyline-apiserver              `https://opendev.org/openstack/skyline-apiserver <https://opendev.org/openstack/skyline-apiserver>`__
 openstack/solum                          `https://opendev.org/openstack/solum <https://opendev.org/openstack/solum>`__
 openstack/storlets                       `https://opendev.org/openstack/storlets <https://opendev.org/openstack/storlets>`__
 openstack/tacker                         `https://opendev.org/openstack/tacker <https://opendev.org/openstack/tacker>`__
+openstack/tap-as-a-service               `https://opendev.org/openstack/tap-as-a-service <https://opendev.org/openstack/tap-as-a-service>`__
 openstack/telemetry-tempest-plugin       `https://opendev.org/openstack/telemetry-tempest-plugin <https://opendev.org/openstack/telemetry-tempest-plugin>`__
-openstack/tricircle                      `https://opendev.org/openstack/tricircle <https://opendev.org/openstack/tricircle>`__
 openstack/trove                          `https://opendev.org/openstack/trove <https://opendev.org/openstack/trove>`__
 openstack/trove-dashboard                `https://opendev.org/openstack/trove-dashboard <https://opendev.org/openstack/trove-dashboard>`__
+openstack/venus                          `https://opendev.org/openstack/venus <https://opendev.org/openstack/venus>`__
+openstack/venus-dashboard                `https://opendev.org/openstack/venus-dashboard <https://opendev.org/openstack/venus-dashboard>`__
 openstack/vitrage                        `https://opendev.org/openstack/vitrage <https://opendev.org/openstack/vitrage>`__
 openstack/vitrage-dashboard              `https://opendev.org/openstack/vitrage-dashboard <https://opendev.org/openstack/vitrage-dashboard>`__
 openstack/vitrage-tempest-plugin         `https://opendev.org/openstack/vitrage-tempest-plugin <https://opendev.org/openstack/vitrage-tempest-plugin>`__
 openstack/watcher                        `https://opendev.org/openstack/watcher <https://opendev.org/openstack/watcher>`__
 openstack/watcher-dashboard              `https://opendev.org/openstack/watcher-dashboard <https://opendev.org/openstack/watcher-dashboard>`__
+openstack/whitebox-tempest-plugin        `https://opendev.org/openstack/whitebox-tempest-plugin <https://opendev.org/openstack/whitebox-tempest-plugin>`__
 openstack/zaqar                          `https://opendev.org/openstack/zaqar <https://opendev.org/openstack/zaqar>`__
 openstack/zaqar-ui                       `https://opendev.org/openstack/zaqar-ui <https://opendev.org/openstack/zaqar-ui>`__
 openstack/zun                            `https://opendev.org/openstack/zun <https://opendev.org/openstack/zun>`__
@@ -135,19 +122,18 @@ starlingx/integ                          `https://opendev.org/starlingx/integ <h
 starlingx/metal                          `https://opendev.org/starlingx/metal <https://opendev.org/starlingx/metal>`__
 starlingx/nfv                            `https://opendev.org/starlingx/nfv <https://opendev.org/starlingx/nfv>`__
 starlingx/update                         `https://opendev.org/starlingx/update <https://opendev.org/starlingx/update>`__
+vexxhost/openstack-operator              `https://opendev.org/vexxhost/openstack-operator <https://opendev.org/vexxhost/openstack-operator>`__
 x/almanach                               `https://opendev.org/x/almanach <https://opendev.org/x/almanach>`__
 x/apmec                                  `https://opendev.org/x/apmec <https://opendev.org/x/apmec>`__
 x/bilean                                 `https://opendev.org/x/bilean <https://opendev.org/x/bilean>`__
 x/broadview-collector                    `https://opendev.org/x/broadview-collector <https://opendev.org/x/broadview-collector>`__
 x/collectd-openstack-plugins             `https://opendev.org/x/collectd-openstack-plugins <https://opendev.org/x/collectd-openstack-plugins>`__
 x/devstack-plugin-additional-pkg-repos   `https://opendev.org/x/devstack-plugin-additional-pkg-repos <https://opendev.org/x/devstack-plugin-additional-pkg-repos>`__
-x/devstack-plugin-bdd                    `https://opendev.org/x/devstack-plugin-bdd <https://opendev.org/x/devstack-plugin-bdd>`__
 x/devstack-plugin-glusterfs              `https://opendev.org/x/devstack-plugin-glusterfs <https://opendev.org/x/devstack-plugin-glusterfs>`__
 x/devstack-plugin-hdfs                   `https://opendev.org/x/devstack-plugin-hdfs <https://opendev.org/x/devstack-plugin-hdfs>`__
 x/devstack-plugin-libvirt-qemu           `https://opendev.org/x/devstack-plugin-libvirt-qemu <https://opendev.org/x/devstack-plugin-libvirt-qemu>`__
 x/devstack-plugin-mariadb                `https://opendev.org/x/devstack-plugin-mariadb <https://opendev.org/x/devstack-plugin-mariadb>`__
-x/devstack-plugin-nfs                    `https://opendev.org/x/devstack-plugin-nfs <https://opendev.org/x/devstack-plugin-nfs>`__
-x/devstack-plugin-sheepdog               `https://opendev.org/x/devstack-plugin-sheepdog <https://opendev.org/x/devstack-plugin-sheepdog>`__
+x/devstack-plugin-tobiko                 `https://opendev.org/x/devstack-plugin-tobiko <https://opendev.org/x/devstack-plugin-tobiko>`__
 x/devstack-plugin-vmax                   `https://opendev.org/x/devstack-plugin-vmax <https://opendev.org/x/devstack-plugin-vmax>`__
 x/drbd-devstack                          `https://opendev.org/x/drbd-devstack <https://opendev.org/x/drbd-devstack>`__
 x/fenix                                  `https://opendev.org/x/fenix <https://opendev.org/x/fenix>`__
@@ -174,6 +160,7 @@ x/networking-fortinet                    `https://opendev.org/x/networking-forti
 x/networking-hpe                         `https://opendev.org/x/networking-hpe <https://opendev.org/x/networking-hpe>`__
 x/networking-huawei                      `https://opendev.org/x/networking-huawei <https://opendev.org/x/networking-huawei>`__
 x/networking-infoblox                    `https://opendev.org/x/networking-infoblox <https://opendev.org/x/networking-infoblox>`__
+x/networking-l2gw                        `https://opendev.org/x/networking-l2gw <https://opendev.org/x/networking-l2gw>`__
 x/networking-lagopus                     `https://opendev.org/x/networking-lagopus <https://opendev.org/x/networking-lagopus>`__
 x/networking-mlnx                        `https://opendev.org/x/networking-mlnx <https://opendev.org/x/networking-mlnx>`__
 x/networking-nec                         `https://opendev.org/x/networking-nec <https://opendev.org/x/networking-nec>`__
@@ -195,10 +182,8 @@ x/rsd-virt-for-nova                      `https://opendev.org/x/rsd-virt-for-nov
 x/scalpels                               `https://opendev.org/x/scalpels <https://opendev.org/x/scalpels>`__
 x/slogging                               `https://opendev.org/x/slogging <https://opendev.org/x/slogging>`__
 x/stackube                               `https://opendev.org/x/stackube <https://opendev.org/x/stackube>`__
-x/tap-as-a-service                       `https://opendev.org/x/tap-as-a-service <https://opendev.org/x/tap-as-a-service>`__
 x/tap-as-a-service-dashboard             `https://opendev.org/x/tap-as-a-service-dashboard <https://opendev.org/x/tap-as-a-service-dashboard>`__
 x/tatu                                   `https://opendev.org/x/tatu <https://opendev.org/x/tatu>`__
-x/tobiko                                 `https://opendev.org/x/tobiko <https://opendev.org/x/tobiko>`__
 x/trio2o                                 `https://opendev.org/x/trio2o <https://opendev.org/x/trio2o>`__
 x/valet                                  `https://opendev.org/x/valet <https://opendev.org/x/valet>`__
 x/vmware-nsx                             `https://opendev.org/x/vmware-nsx <https://opendev.org/x/vmware-nsx>`__
@@ -238,10 +238,10 @@ package dependencies, packages may be listed at the following
 locations in the top-level of the plugin repository:

 - ``./devstack/files/debs/$plugin_name`` - Packages to install when running
-  on Ubuntu, Debian or Linux Mint.
+  on Ubuntu or Debian.

 - ``./devstack/files/rpms/$plugin_name`` - Packages to install when running
-  on Red Hat, Fedora, CentOS or XenServer.
+  on Red Hat, Fedora, or CentOS.

 - ``./devstack/files/rpms-suse/$plugin_name`` - Packages to install when
  running on SUSE Linux or openSUSE.
@@ -196,30 +196,6 @@ See the `remote-pdb`_ home page for more options.

 .. _`remote-pdb`: https://pypi.org/project/remote-pdb/

-Known Issues
-============
-
-Be careful about systemd python libraries. There are 3 of them on
-pypi, and they are all very different. They unfortunately all install
-into the ``systemd`` namespace, which can cause some issues.
-
- ``systemd-python`` - this is the upstream maintained library, it has
-  a version number like systemd itself (currently ``234``). This is
-  the one you want.
- ``systemd`` - a python 3 only library, not what you want.
- ``python-systemd`` - another library you don't want. Installing it
-  on a system will break ansible's ability to run.
-
-
-If we were using user units, the ``[Service]`` - ``Group=`` parameter
-doesn't seem to work with user units, even though the documentation
-says that it should. This means that we will need to do an explicit
-``/usr/bin/sg``. This has the downside of making the SYSLOG_IDENTIFIER
-be ``sg``. We can explicitly set that with ``SyslogIdentifier=``, but
-it's really unfortunate that we're going to need this work
-around. This is currently not a problem because we're only using
-system units.
-
 Future Work
 ===========

@@ -195,12 +195,6 @@ value defined in the parent job, or the default from DevStack, if any.
     - A bridge called br-infra is set up for all jobs that inherit
       from multinode with a dedicated `bridge role
       <https://zuul-ci.org/docs/zuul-jobs/general-roles.html#role-multi-node-bridge>`_.
-   * - DEVSTACK_GATE_FEATURE_MATRIX
-     - devstack-gate
-     - ``test_matrix_features`` variable of the test-matrix role in
-       devstack-gate. This is a temporary solution, feature matrix
-       will go away. In the future services will be defined in jobs
-       only.
   * - DEVSTACK_CINDER_VOLUME_CLEAR
     - devstack
     - *CINDER_VOLUME_CLEAR: true/false* in devstack_localrc in the
@@ -6,7 +6,7 @@ if is_service_enabled tempest; then
        source $TOP_DIR/lib/tempest
    elif [[ "$1" == "stack" && "$2" == "install" ]]; then
        echo_summary "Installing Tempest"
-        install_tempest
+        async_runfunc install_tempest
    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
        # Tempest config must come after layer 2 services are running
        :
@@ -17,6 +17,7 @@ if is_service_enabled tempest; then
        # local.conf Tempest option overrides
        :
    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
+        async_wait install_tempest
        echo_summary "Initializing Tempest"
        configure_tempest
        echo_summary "Installing Tempest Plugins"
@@ -1,5 +1,4 @@
 Listen %PUBLICPORT%
-Listen %ADMINPORT%
 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" keystone_combined

 <Directory %KEYSTONE_BIN%>
@@ -20,20 +19,6 @@ LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)"
    %SSLKEYFILE%
 </VirtualHost>

-<VirtualHost *:%ADMINPORT%>
-    WSGIDaemonProcess keystone-admin processes=3 threads=1 user=%USER% display-name=%{GROUP} %VIRTUALENV%
-    WSGIProcessGroup keystone-admin
-    WSGIScriptAlias / %KEYSTONE_BIN%/keystone-wsgi-admin
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-    ErrorLogFormat "%M"
-    ErrorLog /var/log/%APACHE_NAME%/keystone.log
-    CustomLog /var/log/%APACHE_NAME%/keystone_access.log keystone_combined
-    %SSLENGINE%
-    %SSLCERTFILE%
-    %SSLKEYFILE%
-</VirtualHost>
-
 %SSLLISTEN%<VirtualHost *:443>
 %SSLLISTEN%    %SSLENGINE%
 %SSLLISTEN%    %SSLCERTFILE%
@@ -49,13 +34,3 @@ Alias /identity %KEYSTONE_BIN%/keystone-wsgi-public
    WSGIApplicationGroup %{GLOBAL}
    WSGIPassAuthorization On
 </Location>
-
-Alias /identity_admin %KEYSTONE_BIN%/keystone-wsgi-admin
-<Location /identity_admin>
-    SetHandler wsgi-script
-    Options +ExecCGI
-
-    WSGIProcessGroup keystone-admin
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-</Location>
@@ -1 +0,0 @@
-debs/
@@ -1,6 +1,4 @@
 lvm2
-open-iscsi
-open-iscsi-utils # Deprecated since quantal dist:precise
 qemu-utils
 tgt # NOPRIME
 thin-provisioning-tools
@@ -1,2 +1,2 @@
-dstat
-python-psutil
+dstat # dist:bionic
+pcp
@@ -1,11 +1,11 @@
 apache2
 apache2-dev
 bc
-bridge-utils
 bsdmainutils
 curl
 default-jre-headless  # NOPRIME
 g++
+gawk
 gcc
 gettext  # used for compiling message catalogs
 git
@@ -14,7 +14,6 @@ iputils-ping
 libapache2-mod-proxy-uwsgi
 libffi-dev # for pyOpenSSL
 libjpeg-dev # Pillow 3.0.0
-libmysqlclient-dev  # MySQL-python
 libpcre3-dev # for python-pcre
 libpq-dev  # psycopg2
 libssl-dev # for pyOpenSSL
@@ -27,9 +26,10 @@ openssh-server
 openssl
 pkg-config
 psmisc
-python2.7
-python-dev
-python-gdbm # needed for testr
+python3-dev
+python3-pip
+python3-systemd
+python3-venv
 tar
 tcpdump
 unzip
@@ -2,5 +2,5 @@ libkrb5-dev
 libldap2-dev
 libsasl2-dev
 memcached
-python-mysqldb
+python3-mysqldb
 sqlite3
@@ -1,3 +1,3 @@
 ldap-utils
-python-ldap
+python3-ldap
 slapd
@@ -5,7 +5,7 @@ gir1.2-libosinfo-1.0
 lvm2 # NOPRIME
 netcat-openbsd
 open-iscsi
-python-guestfs # NOPRIME
+python3-guestfs # NOPRIME
 qemu-utils
 sg3-utils
 sysfsutils
@@ -1,15 +1,14 @@
 acl
 dnsmasq-base
-dnsmasq-utils # for dhcp_release only available in dist:precise
+dnsmasq-utils # for dhcp_release
 ebtables
 haproxy # to serve as metadata proxy inside router/dhcp namespaces
 iptables
 iputils-arping
 iputils-ping
-libmysqlclient-dev
 mysql-server #NOPRIME
 postgresql-server-dev-all
-python-mysqldb
+python3-mysqldb
 rabbitmq-server # NOPRIME
 radvd # NOPRIME
 sqlite3
@@ -1,24 +1,18 @@
 conntrack
 curl
-dnsmasq-base
-dnsmasq-utils # for dhcp_release
 ebtables
-gawk
 genisoimage # required for config_drive
 iptables
 iputils-arping
 kpartx
 libjs-jquery-tablesorter # Needed for coverage html reports
-libmysqlclient-dev
-libvirt-bin # dist:xenial NOPRIME
-libvirt-clients # not:xenial NOPRIME
-libvirt-daemon-system # not:xenial NOPRIME
+libvirt-clients # NOPRIME
+libvirt-daemon-system # NOPRIME
 libvirt-dev # NOPRIME
 mysql-server # NOPRIME
 parted
 pm-utils
-python-mysqldb
-qemu # dist:wheezy,jessie NOPRIME
+python3-mysqldb
 qemu-kvm # NOPRIME
 rabbitmq-server # NOPRIME
 socat # used by ajaxterm
@@ -0,0 +1,3 @@
+lsscsi
+open-iscsi
+open-iscsi-utils # Deprecated since quantal dist:precise
@@ -0,0 +1,3 @@
+ovn-central
+ovn-controller-vtep
+ovn-host
@@ -2,5 +2,6 @@ curl
 liberasurecode-dev
 make
 memcached
+rsync
 sqlite3
 xfsprogs
@@ -1,4 +1,4 @@
-dn: olcDatabase={${LDAP_OLCDB_NUMBER}}hdb,cn=config
+dn: olcDatabase={${LDAP_OLCDB_NUMBER}}${LDAP_OLCDB_TYPE},cn=config
 changetype: modify
 replace: olcSuffix
 olcSuffix: ${BASE_DN}
@@ -0,0 +1,16 @@
+[Unit]
+Description=Activate LVM backing file %BACKING_FILE%
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+ExecStart=/sbin/losetup --find --show %DIRECTIO% %BACKING_FILE%
+ExecStop=/bin/sh -c '/sbin/losetup -d $$(/sbin/losetup --associated %BACKING_FILE% -O NAME -n)'
+RemainAfterExit=yes
+Type=oneshot
+
+[Install]
+WantedBy=local-fs.target
+Also=systemd-udev-settle.service
@@ -1,4 +1,3 @@
 lvm2
-open-iscsi
 qemu-tools
 tgt # NOPRIME
@@ -1,2 +1 @@
 dstat
-python-psutil
@@ -1,9 +1,9 @@
 apache2
 apache2-devel
 bc
-bridge-utils
 ca-certificates-mozilla
 curl
+gawk
 gcc
 gcc-c++
 git-core
@@ -21,10 +21,10 @@ openssl
 pcre-devel # python-pcre
 postgresql-devel  # psycopg2
 psmisc
+python3-systemd
 python-cmd2 # dist:opensuse-12.3
 python-devel  # pyOpenSSL
 python-xml
-systemd-devel # for systemd-python
 tar
 tcpdump
 unzip
@@ -5,7 +5,6 @@ ebtables
 haproxy # to serve as metadata proxy inside router/dhcp namespaces
 iptables
 iputils
-mariadb # NOPRIME
 rabbitmq-server # NOPRIME
 radvd # NOPRIME
 sqlite3
@@ -1,17 +1,13 @@
 cdrkit-cdrtools-compat # dist:sle12
 conntrack-tools
 curl
-dnsmasq
-dnsmasq-utils # dist:opensuse-12.3,opensuse-13.1
 ebtables
-gawk
 iptables
 iputils
 kpartx
 kvm # NOPRIME
 libvirt # NOPRIME
 libvirt-python # NOPRIME
-mariadb # NOPRIME
 # mkisofs is required for config_drive
 mkisofs # not:sle12
 parted
@@ -0,0 +1,2 @@
+lsscsi
+open-iscsi
@@ -1,3 +1,3 @@
 ceph    # NOPRIME
-redhat-lsb-core
+redhat-lsb-core # not:rhel9
 xfsprogs
@@ -1,5 +1,3 @@
-iscsi-initiator-utils
 lvm2
 qemu-img
-scsi-target-utils # not:rhel7,f25,f26,f27,f28,f29 NOPRIME
-targetcli # dist:rhel7,f25,f26,f27,f28,f29 NOPRIME
+targetcli
@@ -1,3 +1 @@
-dstat # not:f29
-pcp-system-tools # dist:f29
-python-psutil
+pcp-system-tools
@@ -1,7 +1,7 @@
 bc
-bridge-utils
 curl
 dbus
+gawk
 gcc
 gcc-c++
 gettext  # used for compiling message catalogs
@@ -9,15 +9,14 @@ git-core
 graphviz # needed only for docs
 httpd
 httpd-devel
-iptables-services  # NOPRIME f25,f26,f27,f28,f29
-java-1.7.0-openjdk-headless  # NOPRIME rhel7
-java-1.8.0-openjdk-headless  # NOPRIME f25,f26,f27,f28,f29
+iptables-services
+java-1.8.0-openjdk-headless
 libffi-devel
 libjpeg-turbo-devel # Pillow 3.0.0
 libxml2-devel # lxml
 libxslt-devel # lxml
 libyaml-devel
-mariadb-devel  # MySQL-python
+mod_ssl # required for tls-proxy on centos 9 stream computes
 net-tools
 openssh-server
 openssl
@@ -26,10 +25,10 @@ pcre-devel # for python-pcre
 pkgconfig
 postgresql-devel  # psycopg2
 psmisc
-pyOpenSSL # version in pip uses too much memory
-python-devel
+python3-devel
+python3-pip
+python3-systemd
 redhat-rpm-config # missing dep for gcc hardening flags, see rhbz#1217376
-systemd-devel # for systemd-python
 tar
 tcpdump
 unzip
@@ -1,9 +1,10 @@
 cryptsetup
 dosfstools
-genisoimage
+genisoimage # not:rhel9
 iscsi-initiator-utils
 libosinfo
 lvm2
 sg3_utils
 # Stuff for diablo volumes
 sysfsutils
+xorriso # not:rhel8
@@ -5,8 +5,6 @@ ebtables
 haproxy # to serve as metadata proxy inside router/dhcp namespaces
 iptables
 iputils
-mysql-devel
-mysql-server # NOPRIME
 openvswitch # NOPRIME
 rabbitmq-server # NOPRIME
 radvd # NOPRIME
@@ -1,20 +1,14 @@
 conntrack-tools
 curl
-dnsmasq # for q-dhcp
-dnsmasq-utils # for dhcp_release
 ebtables
-gawk
-genisoimage # required for config_drive
+genisoimage # not:rhel9 required for config_drive
 iptables
 iputils
-kernel-modules # dist:f25,f26,f27,f28,f29
+kernel-modules
 kpartx
-libxml2-python
-m2crypto
-mysql-devel
-mysql-server # NOPRIME
 parted
 polkit
 rabbitmq-server # NOPRIME
 sqlite
 sudo
+xorriso # not:rhel8
@@ -0,0 +1,2 @@
+iscsi-initiator-utils
+lsscsi
@@ -0,0 +1,3 @@
+ovn-central
+ovn-host
+ovn-vtep
@@ -1,8 +1,7 @@
 curl
 liberasurecode-devel
 memcached
-pyxattr
-rsync-daemon # dist:f25,f26,f27,f28,f29
+rsync-daemon
 sqlite
 xfsprogs
-xinetd
+xinetd # not:f35,rhel9
@@ -21,6 +21,7 @@ source ${FUNC_DIR}/inc/ini-config
 source ${FUNC_DIR}/inc/meta-config
 source ${FUNC_DIR}/inc/python
 source ${FUNC_DIR}/inc/rootwrap
+source ${FUNC_DIR}/inc/async

 # Save trace setting
 _XTRACE_FUNCTIONS=$(set +o | grep xtrace)
@@ -77,6 +78,48 @@ function get_extra_file {
    fi
 }

+# Generate image property arguments for OSC
+#
+# Arguments: properties, one per, like propname=value
+#
+# Result is --property propname1=value1 --property propname2=value2
+function _image_properties_to_arg {
+    local result=""
+    for property in $*; do
+        result+=" --property $property"
+    done
+    echo $result
+}
+
+# Upload an image to glance using the configured mechanism
+#
+# Arguments:
+#  image name
+#  container format
+#  disk format
+#  path to image file
+#  optional properties (format of propname=value)
+#
+function _upload_image {
+    local image_name="$1"
+    shift
+    local container="$1"
+    shift
+    local disk="$1"
+    shift
+    local image="$1"
+    shift
+    local properties
+    local useimport
+
+    properties=$(_image_properties_to_arg $*)
+
+    if [[ "$GLANCE_USE_IMPORT_WORKFLOW" == "True" ]]; then
+        useimport="--import"
+    fi
+
+    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties < "${image}"
+}

 # Retrieve an image from a URL and upload into Glance.
 # Uses the following variables:
@@ -118,7 +161,7 @@ function upload_image {
    # OpenVZ-format images are provided as .tar.gz, but not decompressed prior to loading
    if [[ "$image_url" =~ 'openvz' ]]; then
        image_name="${image_fname%.tar.gz}"
-        openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format ami --disk-format ami < "${image}"
+        _upload_image "$image_name" ami ami "$image"
        return
    fi

@@ -232,42 +275,8 @@ function upload_image {
        vmdk_adapter_type="${props[1]:-$vmdk_adapter_type}"
        vmdk_net_adapter="${props[2]:-$vmdk_net_adapter}"

-        openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format bare --disk-format vmdk --property vmware_disktype="$vmdk_disktype" --property vmware_adaptertype="$vmdk_adapter_type" --property hw_vif_model="$vmdk_net_adapter" < "${image}"
-        return
-    fi
+        _upload_image "$image_name" bare vmdk "$image" vmware_disktype="$vmdk_disktype" vmware_adaptertype="$vmdk_adapter_type" hw_vif_model="$vmdk_net_adapter"

-    # XenServer-vhd-ovf-format images are provided as .vhd.tgz
-    # and should not be decompressed prior to loading
-    if [[ "$image_url" =~ '.vhd.tgz' ]]; then
-        image_name="${image_fname%.vhd.tgz}"
-        local force_vm_mode=""
-        if [[ "$image_name" =~ 'cirros' ]]; then
-            # Cirros VHD image currently only boots in PV mode.
-            # Nova defaults to PV for all VHD images, but
-            # the glance setting is needed for booting
-            # directly from volume.
-            force_vm_mode="--property vm_mode=xen"
-        fi
-        openstack \
-            --os-cloud=devstack-admin --os-region-name="$REGION_NAME" \
-            image create \
-            "$image_name" --public \
-            --container-format=ovf --disk-format=vhd \
-            $force_vm_mode < "${image}"
-        return
-    fi
-
-    # .xen-raw.tgz suggests a Xen capable raw image inside a tgz.
-    # and should not be decompressed prior to loading.
-    # Setting metadata, so PV mode is used.
-    if [[ "$image_url" =~ '.xen-raw.tgz' ]]; then
-        image_name="${image_fname%.xen-raw.tgz}"
-        openstack \
-            --os-cloud=devstack-admin --os-region-name="$REGION_NAME" \
-            image create \
-            "$image_name" --public \
-            --container-format=tgz --disk-format=raw \
-            --property vm_mode=xen < "${image}"
        return
    fi

@@ -278,12 +287,7 @@ function upload_image {
            die $LINENO "Unknown vm_mode=${vm_mode} for Virtuozzo image"
        fi

-        openstack \
-            --os-cloud=devstack-admin --os-region-name="$REGION_NAME" \
-            image create \
-            "$image_name" --public \
-            --container-format=bare --disk-format=ploop \
-            --property vm_mode=$vm_mode < "${image}"
+        _upload_image "$image_name" bare ploop "$image" vm_mode=$vm_mode
        return
    fi

@@ -293,6 +297,15 @@ function upload_image {
    local container_format=""
    local unpack=""
    local img_property=""
+
+    # NOTE(danms): If we're on libvirt/qemu or libvirt/kvm, set the hw_rng_model
+    # to libvirt in the image properties.
+    if [[ "$VIRT_DRIVER" == "libvirt" ]]; then
+        if [[ "$LIBVIRT_TYPE" == "qemu" || "$LIBVIRT_TYPE" == "kvm" ]]; then
+            img_property="hw_rng_model=virtio"
+        fi
+    fi
+
    case "$image_fname" in
        *.tar.gz|*.tgz)
            # Extract ami and aki files
@@ -341,6 +354,12 @@ function upload_image {
            disk_format=qcow2
            container_format=bare
            ;;
+        *.qcow2.xz)
+            image_name=$(basename "$image" ".qcow2.xz")
+            disk_format=qcow2
+            container_format=bare
+            unpack=unxz
+            ;;
        *.raw)
            image_name=$(basename "$image" ".raw")
            disk_format=raw
@@ -364,20 +383,30 @@ function upload_image {
    esac

    if is_arch "ppc64le" || is_arch "ppc64" || is_arch "ppc"; then
-        img_property="--property hw_cdrom_bus=scsi --property os_command_line=console=hvc0"
+        img_property="$img_property hw_cdrom_bus=scsi os_command_line=console=hvc0"
    fi

    if is_arch "aarch64"; then
-        img_property="--property hw_machine_type=virt --property hw_cdrom_bus=scsi --property hw_scsi_model=virtio-scsi --property os_command_line='console=ttyAMA0'"
+        img_property="$img_property hw_machine_type=virt hw_cdrom_bus=scsi hw_scsi_model=virtio-scsi os_command_line='console=ttyAMA0'"
    fi

    if [ "$container_format" = "bare" ]; then
        if [ "$unpack" = "zcat" ]; then
-            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < <(zcat --force "${image}")
+            _upload_image "$image_name" $container_format $disk_format <(zcat --force "$image") $img_property
        elif [ "$unpack" = "bunzip2" ]; then
-            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < <(bunzip2 -cdk "${image}")
+            _upload_image "$image_name" $container_format $disk_format <(bunzip2 -cdk "$image") $img_property
+        elif [ "$unpack" = "unxz" ]; then
+            # NOTE(brtknr): unxz the file first and cleanup afterwards to
+            # prevent timeout while Glance tries to upload image (e.g. to Swift).
+            local tmp_dir
+            local image_path
+            tmp_dir=$(mktemp -d)
+            image_path="$tmp_dir/$image_name"
+            unxz -cv "${image}" > "$image_path"
+            _upload_image "$image_name" $container_format $disk_format "$image_path" $img_property
+            rm -rf $tmp_dir
        else
-            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < "${image}"
+            _upload_image "$image_name" $container_format $disk_format "$image" $img_property
        fi
    else
        # Use glance client to add the kernel the root filesystem.
@@ -385,12 +414,12 @@ function upload_image {
        # kernel for use when uploading the root filesystem.
        local kernel_id="" ramdisk_id="";
        if [ -n "$kernel" ]; then
-            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $img_property --public --container-format aki --disk-format aki < "$kernel" | grep ' id ' | get_field 2)
+            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki < "$kernel" -f value -c id)
        fi
        if [ -n "$ramdisk" ]; then
-            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $img_property --public --container-format ari --disk-format ari < "$ramdisk" | grep ' id ' | get_field 2)
+            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari < "$ramdisk" -f value -c id)
        fi
-        openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "${image_name%.img}" $img_property --public --container-format ami --disk-format ami ${kernel_id:+--property kernel_id=$kernel_id} ${ramdisk_id:+--property ramdisk_id=$ramdisk_id} < "${image}"
+        _upload_image "${image_name%.img}" ami ami "$image" ${kernel_id:+ kernel_id=$kernel_id} ${ramdisk_id:+ ramdisk_id=$ramdisk_id} $img_property
    fi
 }

@@ -635,40 +664,29 @@ function vercmp {
 # This sets up defaults we like in devstack for logging for tracking
 # down issues, and makes sure everything is done the same between
 # projects.
+# NOTE(jh): Historically this function switched between three different
+# functions: setup_systemd_logging, setup_colorized_logging and
+# setup_standard_logging_identity. Since we always run with systemd now,
+# this could be cleaned up, but the other functions may still be in use
+# by plugins. Since deprecations haven't worked in the past, we'll just
+# leave them in place.
 function setup_logging {
-    local conf_file=$1
-    local other_cond=${2:-"False"}
-    if [[ "$USE_SYSTEMD" == "True" ]]; then
-        setup_systemd_logging $conf_file
-    elif [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] && [ "$other_cond" == "False" ]; then
-        setup_colorized_logging $conf_file
-    else
-        setup_standard_logging_identity $conf_file
-    fi
+    setup_systemd_logging $1
 }

 # This function sets log formatting options for colorizing log
 # output to stdout. It is meant to be called by lib modules.
-# The last two parameters are optional and can be used to specify
-# non-default value for project and user format variables.
-# Defaults are respectively 'project_name' and 'user_name'
-#
-# setup_colorized_logging something.conf SOMESECTION
 function setup_colorized_logging {
    local conf_file=$1
-    local conf_section="DEFAULT"
-    local project_var="project_name"
-    local user_var="user_name"
    # Add color to logging output
-    iniset $conf_file $conf_section logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[01;36m%(request_id)s [00;36m%("$project_var")s %("$user_var")s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
-    iniset $conf_file $conf_section logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
-    iniset $conf_file $conf_section logging_debug_format_suffix "[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
-    iniset $conf_file $conf_section logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s [01;35m%(instance)s[00m"
+    iniset $conf_file DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[01;36m%(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
+    iniset $conf_file DEFAULT logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
+    iniset $conf_file DEFAULT logging_debug_format_suffix "[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
+    iniset $conf_file DEFAULT logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s [01;35m%(instance)s[00m"
 }

 function setup_systemd_logging {
    local conf_file=$1
-    local conf_section="DEFAULT"
    # NOTE(sdague): this is a nice to have, and means we're using the
    # native systemd path, which provides for things like search on
    # request-id. However, there may be an eventlet interaction here,
@@ -676,16 +694,16 @@ function setup_systemd_logging {
    USE_JOURNAL=$(trueorfalse False USE_JOURNAL)
    local pidstr=""
    if [[ "$USE_JOURNAL" == "True" ]]; then
-        iniset $conf_file $conf_section use_journal "True"
+        iniset $conf_file DEFAULT use_journal "True"
        # if we are using the journal directly, our process id is already correct
    else
        pidstr="(pid=%(process)d) "
    fi
-    iniset $conf_file $conf_section logging_debug_format_suffix "[00;33m{{${pidstr}%(funcName)s %(pathname)s:%(lineno)d}}[00m"
+    iniset $conf_file DEFAULT logging_debug_format_suffix "[00;33m{{${pidstr}%(funcName)s %(pathname)s:%(lineno)d}}[00m"

-    iniset $conf_file $conf_section logging_context_format_string "%(color)s%(levelname)s %(name)s [[01;36m%(global_request_id)s %(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
-    iniset $conf_file $conf_section logging_default_format_string "%(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
-    iniset $conf_file $conf_section logging_exception_prefix "ERROR %(name)s [01;35m%(instance)s[00m"
+    iniset $conf_file DEFAULT logging_context_format_string "%(color)s%(levelname)s %(name)s [[01;36m%(global_request_id)s %(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
+    iniset $conf_file DEFAULT logging_default_format_string "%(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
+    iniset $conf_file DEFAULT logging_exception_prefix "ERROR %(name)s [01;35m%(instance)s[00m"
 }

 function setup_standard_logging_identity {
@@ -709,23 +727,22 @@ if ! function_exists echo_nolog; then
 fi


-# create_disk - Create backing disk
+# create_disk - Create, configure, and mount a backing disk
 function create_disk {
    local node_number
    local disk_image=${1}
    local storage_data_dir=${2}
    local loopback_disk_size=${3}
+    local key

-    # Create a loopback disk and format it to XFS.
-    if [[ -e ${disk_image} ]]; then
-        if egrep -q ${storage_data_dir} /proc/mounts; then
-            sudo umount ${storage_data_dir}
-            sudo rm -f ${disk_image}
-        fi
-    fi
+    key=$(echo $disk_image | sed 's#/.##')
+    key="devstack-$key"

-    sudo mkdir -p ${storage_data_dir}/drives/images
+    destroy_disk $disk_image $storage_data_dir

+    # Create an empty file of the correct size (and ensure the
+    # directory structure up to that path exists)
+    sudo mkdir -p $(dirname ${disk_image})
    sudo truncate -s ${loopback_disk_size} ${disk_image}

    # Make a fresh XFS filesystem. Use bigger inodes so xattr can fit in
@@ -735,11 +752,31 @@ function create_disk {
    # Swift and Ceph.
    sudo mkfs.xfs -f -i size=1024 ${disk_image}

-    # Mount the disk with mount options to make it as efficient as possible
-    if ! egrep -q ${storage_data_dir} /proc/mounts; then
-        sudo mount -t xfs -o loop,noatime,nodiratime,logbufs=8  \
-            ${disk_image} ${storage_data_dir}
+    # Install a new loopback fstab entry for this disk image, and mount it
+    echo "$disk_image $storage_data_dir xfs loop,noatime,nodiratime,logbufs=8,comment=$key 0 0" | sudo tee -a /etc/fstab
+    sudo mkdir -p $storage_data_dir
+    sudo mount -v $storage_data_dir
+}
+
+# Unmount, de-configure, and destroy a backing disk
+function destroy_disk {
+    local disk_image=$1
+    local storage_data_dir=$2
+    local key
+
+    key=$(echo $disk_image | sed 's#/.##')
+    key="devstack-$key"
+
+    # Unmount the target, if mounted
+    if egrep -q $storage_data_dir /proc/mounts; then
+        sudo umount $storage_data_dir
    fi
+
+    # Clear any fstab rules
+    sudo sed -i '/.*comment=$key.*/ d' /etc/fstab
+
+    # Delete the file
+    sudo rm -f $disk_image
 }


@@ -27,7 +27,6 @@
 # - ``RECLONE``
 # - ``REQUIREMENTS_DIR``
 # - ``STACK_USER``
-# - ``TRACK_DEPENDS``
 # - ``http_proxy``, ``https_proxy``, ``no_proxy``
 #

@@ -44,14 +43,13 @@ declare -A -g GITREPO
 declare -A -g GITBRANCH
 declare -A -g GITDIR

-TRACK_DEPENDS=${TRACK_DEPENDS:-False}
 KILL_PATH="$(which kill)"

 # Save these variables to .stackenv
 STACK_ENV_VARS="BASE_SQL_CONN DATA_DIR DEST ENABLED_SERVICES HOST_IP \
-    KEYSTONE_AUTH_URI KEYSTONE_SERVICE_URI \
+    KEYSTONE_SERVICE_URI \
    LOGFILE OS_CACERT SERVICE_HOST STACK_USER TLS_IP \
-    HOST_IPV6 SERVICE_IP_VERSION"
+    HOST_IPV6 SERVICE_IP_VERSION TUNNEL_ENDPOINT_IP TUNNEL_IP_VERSION"


 # Saves significant environment variables to .stackenv for later use
@@ -87,7 +85,7 @@ function write_clouds_yaml {
    if [ -f "$SSL_BUNDLE_FILE" ]; then
        CA_CERT_ARG="--os-cacert $SSL_BUNDLE_FILE"
    fi
-    # demo -> devstack
+    # devstack: user with the member role on demo project
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack \
@@ -98,18 +96,7 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-project-name demo

-    # alt_demo -> devstack-alt
-    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
-        --file $CLOUDS_YAML \
-        --os-cloud devstack-alt \
-        --os-region-name $REGION_NAME \
-        $CA_CERT_ARG \
-        --os-auth-url $KEYSTONE_SERVICE_URI \
-        --os-username alt_demo \
-        --os-password $ADMIN_PASSWORD \
-        --os-project-name alt_demo
-
-    # admin -> devstack-admin
+    # devstack-admin: user with the admin role on the admin project
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack-admin \
@@ -120,7 +107,62 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-project-name admin

-    # admin with a system-scoped token -> devstack-system
+    # devstack-admin-demo: user with the admin role on the demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-admin-demo \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username admin \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name demo
+
+    # devstack-alt: user with the member role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-alt-member: user with the member role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt-member \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo_member \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-alt-reader: user with the reader role on alt_demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-alt-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username alt_demo_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name alt_demo
+
+    # devstack-reader: user with the reader role on demo project
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username demo_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-project-name demo
+
+    # devstack-system-admin: user with the admin role on the system
    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
        --file $CLOUDS_YAML \
        --os-cloud devstack-system-admin \
@@ -131,6 +173,33 @@ function write_clouds_yaml {
        --os-password $ADMIN_PASSWORD \
        --os-system-scope all

+    # devstack-system-member: user with the member role on the system
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-system-member \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username system_member \
+        --os-password $ADMIN_PASSWORD \
+        --os-system-scope all
+
+    # devstack-system-reader: user with the reader role on the system
+    $PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
+        --file $CLOUDS_YAML \
+        --os-cloud devstack-system-reader \
+        --os-region-name $REGION_NAME \
+        $CA_CERT_ARG \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username system_reader \
+        --os-password $ADMIN_PASSWORD \
+        --os-system-scope all
+
+    cat >> $CLOUDS_YAML <<EOF
+functional:
+  image_name: $DEFAULT_IMAGE_NAME
+EOF
+
    # CLean up any old clouds.yaml files we had laying around
    rm -f $(eval echo ~"$STACK_USER")/.config/openstack/clouds.yaml
 }
@@ -331,9 +400,6 @@ function _ensure_lsb_release {
        sudo zypper -n install lsb-release
    elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
        sudo dnf install -y redhat-lsb-core
-    elif [[ -x $(command -v yum 2>/dev/null) ]]; then
-        # all rh patforms (fedora, centos, rhel) have this pkg
-        sudo yum install -y redhat-lsb-core
    else
        die $LINENO "Unable to find or auto-install lsb_release"
    fi
@@ -346,14 +412,21 @@ function _ensure_lsb_release {
 #  - os_VENDOR
 #  - os_PACKAGE
 function GetOSVersion {
-    # We only support distros that provide a sane lsb_release
-    _ensure_lsb_release
+    # CentOS Stream 9 does not provide lsb_release
+    source /etc/os-release
+    if [[ "${ID}${VERSION}" == "centos9" ]]; then
+        os_RELEASE=${VERSION_ID}
+        os_CODENAME="n/a"
+        os_VENDOR=$(echo $NAME | tr -d '[:space:]')
+    else
+        _ensure_lsb_release

-    os_RELEASE=$(lsb_release -r -s)
-    os_CODENAME=$(lsb_release -c -s)
-    os_VENDOR=$(lsb_release -i -s)
+        os_RELEASE=$(lsb_release -r -s)
+        os_CODENAME=$(lsb_release -c -s)
+        os_VENDOR=$(lsb_release -i -s)
+    fi

-    if [[ $os_VENDOR =~ (Debian|Ubuntu|LinuxMint) ]]; then
+    if [[ $os_VENDOR =~ (Debian|Ubuntu) ]]; then
        os_PACKAGE="deb"
    else
        os_PACKAGE="rpm"
@@ -371,9 +444,8 @@ declare -g DISTRO

 function GetDistro {
    GetOSVersion
-    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) || \
-            "$os_VENDOR" =~ (LinuxMint) ]]; then
-        # 'Everyone' refers to Ubuntu / Debian / Mint releases by
+    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
+        # 'Everyone' refers to Ubuntu / Debian releases by
        # the code name adjective
        DISTRO=$os_CODENAME
    elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
@@ -391,14 +463,13 @@ function GetDistro {
        DISTRO="sle${os_RELEASE%.*}"
    elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
        "$os_VENDOR" =~ (CentOS) || \
+        "$os_VENDOR" =~ (AlmaLinux) || \
        "$os_VENDOR" =~ (Scientific) || \
        "$os_VENDOR" =~ (OracleServer) || \
        "$os_VENDOR" =~ (Virtuozzo) ]]; then
        # Drop the . release as we assume it's compatible
        # XXX re-evaluate when we get RHEL10
        DISTRO="rhel${os_RELEASE::1}"
-    elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
-        DISTRO="xs${os_RELEASE%.*}"
    else
        # We can't make a good choice here.  Setting a sensible DISTRO
        # is part of the problem, but not the major issue -- we really
@@ -451,8 +522,10 @@ function is_fedora {

    [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
        [ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
-        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleServer" ] || \
-        [ "$os_VENDOR" = "Virtuozzo" ]
+        [ "$os_VENDOR" = "RedHatEnterprise" ] || \
+        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \
+        [ "$os_VENDOR" = "AlmaLinux" ] || \
+        [ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ]
 }


@@ -497,7 +570,6 @@ function is_ubuntu {
    [ "$os_PACKAGE" = "deb" ]
 }

-
 # Git Functions
 # =============

@@ -548,7 +620,7 @@ function git_clone {
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to 'required-projects' in the job definition."
-                die $LINENO "Cloning not allowed in this configuration"
+                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
            git_timed clone $git_clone_flags $git_remote $git_dest
        fi
@@ -560,10 +632,12 @@ function git_clone {
            if [[ "$ERROR_ON_CLONE" = "True" ]]; then
                echo "The $git_dest project was not found; if this is a gate job, add"
                echo "the project to the \$PROJECTS variable in the job definition."
-                die $LINENO "Cloning not allowed in this configuration"
+                die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
            fi
-            # '--branch' can also take tags
-            git_timed clone $git_clone_flags $git_remote $git_dest --branch $git_ref
+            git_timed clone --no-checkout $git_clone_flags $git_remote $git_dest
+            cd $git_dest
+            git_timed fetch $git_clone_flags origin $git_ref
+            git_timed checkout FETCH_HEAD
        elif [[ "$RECLONE" = "True" ]]; then
            # if it does exist then simulate what clone does if asked to RECLONE
            cd $git_dest
@@ -573,7 +647,7 @@ function git_clone {
            # remove the existing ignored files (like pyc) as they cause breakage
            # (due to the py files having older timestamps than our pyc, so python
            # thinks the pyc files are correct using them)
-            find $git_dest -name '*.pyc' -delete
+            sudo find $git_dest -name '*.pyc' -delete

            # handle git_ref accordingly to type (tag, branch)
            if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
@@ -589,6 +663,18 @@ function git_clone {
        fi
    fi

+    # NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
+    # about how we clone and work with repos.  Mark them safe globally
+    # as a work-around.
+    #
+    # NOTE(danms): On bionic (and likely others) git-config may write
+    # ~stackuser/.gitconfig if not run with sudo -H. Using --system
+    # writes these changes to /etc/gitconfig which is more
+    # discoverable anyway.
+    #
+    # [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
+    sudo git config --system --add safe.directory ${git_dest}
+
    # print out the results so we know what change was used in the logs
    cd $git_dest
    git show --oneline | head -1
@@ -794,10 +880,10 @@ function get_or_create_domain {
    # Gets domain id
    domain_id=$(
        # Gets domain id
-        openstack domain show $1 \
+        openstack --os-cloud devstack-system-admin domain show $1 \
            -f value -c id 2>/dev/null ||
        # Creates new domain
-        openstack domain create $1 \
+        openstack --os-cloud devstack-system-admin domain create $1 \
            --description "$2" \
            -f value -c id
    )
@@ -812,7 +898,7 @@ function get_or_create_group {
    # Gets group id
    group_id=$(
        # Creates new group with --or-show
-        openstack group create $1 \
+        openstack --os-cloud devstack-system-admin group create $1 \
            --domain $2 --description "$desc" --or-show \
            -f value -c id
    )
@@ -831,7 +917,7 @@ function get_or_create_user {
    # Gets user id
    user_id=$(
        # Creates new user with --or-show
-        openstack user create \
+        openstack --os-cloud devstack-system-admin user create \
            $1 \
            --password "$2" \
            --domain=$3 \
@@ -848,7 +934,7 @@ function get_or_create_project {
    local project_id
    project_id=$(
        # Creates new project with --or-show
-        openstack project create $1 \
+        openstack --os-cloud devstack-system-admin project create $1 \
            --domain=$2 \
            --or-show -f value -c id
    )
@@ -861,7 +947,7 @@ function get_or_create_role {
    local role_id
    role_id=$(
        # Creates role with --or-show
-        openstack role create $1 \
+        openstack --os-cloud devstack-system-admin role create $1 \
            --or-show -f value -c id
    )
    echo $role_id
@@ -891,7 +977,7 @@ function get_or_add_user_project_role {
    domain_args=$(_get_domain_args $4 $5)

    # Gets user role id
-    user_role_id=$(openstack role assignment list \
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --project $3 \
@@ -899,11 +985,11 @@ function get_or_add_user_project_role {
        | grep '^|\s[a-f0-9]\+' | get_field 1)
    if [[ -z "$user_role_id" ]]; then
        # Adds role to user and get it
-        openstack role add $1 \
+        openstack --os-cloud devstack-system-admin role add $1 \
            --user $2 \
            --project $3 \
            $domain_args
-        user_role_id=$(openstack role assignment list \
+        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
            --role $1 \
            --user $2 \
            --project $3 \
@@ -918,17 +1004,17 @@ function get_or_add_user_project_role {
 function get_or_add_user_domain_role {
    local user_role_id
    # Gets user role id
-    user_role_id=$(openstack role assignment list \
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --user $2 \
        --domain $3 \
        | grep '^|\s[a-f0-9]\+' | get_field 1)
    if [[ -z "$user_role_id" ]]; then
        # Adds role to user and get it
-        openstack role add $1 \
+        openstack --os-cloud devstack-system-admin role add $1 \
            --user $2 \
            --domain $3
-        user_role_id=$(openstack role assignment list \
+        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
            --role $1 \
            --user $2 \
            --domain $3 \
@@ -937,22 +1023,53 @@ function get_or_add_user_domain_role {
    echo $user_role_id
 }

+# Gets or adds user role to system
+# Usage: get_or_add_user_system_role <role> <user> <system> [<user_domain>]
+function get_or_add_user_system_role {
+    local user_role_id
+    local domain_args
+
+    domain_args=$(_get_domain_args $4)
+
+    # Gets user role id
+    user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
+        --role $1 \
+        --user $2 \
+        --system $3 \
+        $domain_args \
+        -f value -c Role)
+    if [[ -z "$user_role_id" ]]; then
+        # Adds role to user and get it
+        openstack --os-cloud devstack-system-admin role add $1 \
+            --user $2 \
+            --system $3 \
+            $domain_args
+        user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
+            --role $1 \
+            --user $2 \
+            --system $3 \
+            $domain_args \
+            -f value -c Role)
+    fi
+    echo $user_role_id
+}
+
 # Gets or adds group role to project
 # Usage: get_or_add_group_project_role <role> <group> <project>
 function get_or_add_group_project_role {
    local group_role_id
    # Gets group role id
-    group_role_id=$(openstack role assignment list \
+    group_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
        --role $1 \
        --group $2 \
        --project $3 \
        -f value)
    if [[ -z "$group_role_id" ]]; then
        # Adds role to group and get it
-        openstack role add $1 \
+        openstack --os-cloud devstack-system-admin role add $1 \
            --group $2 \
            --project $3
-        group_role_id=$(openstack role assignment list \
+        group_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
            --role $1 \
            --group $2 \
            --project $3 \
@@ -968,9 +1085,9 @@ function get_or_create_service {
    # Gets service id
    service_id=$(
        # Gets service id
-        openstack service show $2 -f value -c id 2>/dev/null ||
+        openstack --os-cloud devstack-system-admin service show $2 -f value -c id 2>/dev/null ||
        # Creates new service if not exists
-        openstack service create \
+        openstack --os-cloud devstack-system-admin service create \
            $2 \
            --name $1 \
            --description="$3" \
@@ -983,14 +1100,14 @@ function get_or_create_service {
 # Usage: _get_or_create_endpoint_with_interface <service> <interface> <url> <region>
 function _get_or_create_endpoint_with_interface {
    local endpoint_id
-    endpoint_id=$(openstack endpoint list \
+    endpoint_id=$(openstack --os-cloud devstack-system-admin endpoint list \
        --service $1 \
        --interface $2 \
        --region $4 \
        -c ID -f value)
    if [[ -z "$endpoint_id" ]]; then
        # Creates new endpoint
-        endpoint_id=$(openstack endpoint create \
+        endpoint_id=$(openstack --os-cloud devstack-system-admin endpoint create \
            $1 $2 $3 --region $4 -f value -c id)
    fi

@@ -1024,7 +1141,7 @@ function get_or_create_endpoint {
 # Get a URL from the identity service
 # Usage: get_endpoint_url <service> <interface>
 function get_endpoint_url {
-    echo $(openstack endpoint list \
+    echo $(openstack --os-cloud devstack-system-admin endpoint list \
            --service $1 --interface $2 \
            -c URL -f value)
 }
@@ -1038,6 +1155,11 @@ function is_ironic_hardware {
    return 1
 }

+function is_ironic_enforce_scope {
+    is_service_enabled ironic && [[ "$IRONIC_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]] && return 0
+    return 1
+}
+

 # Package Functions
 # =================
@@ -1219,10 +1341,16 @@ function get_packages {
            if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
                file_to_parse="${file_to_parse} ${package_dir}/glance"
            fi
+            if [[ ! $file_to_parse =~ $package_dir/os-brick ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/os-brick"
+            fi
        elif [[ $service == c-* ]]; then
            if [[ ! $file_to_parse =~ $package_dir/cinder ]]; then
                file_to_parse="${file_to_parse} ${package_dir}/cinder"
            fi
+            if [[ ! $file_to_parse =~ $package_dir/os-brick ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/os-brick"
+            fi
        elif [[ $service == s-* ]]; then
            if [[ ! $file_to_parse =~ $package_dir/swift ]]; then
                file_to_parse="${file_to_parse} ${package_dir}/swift"
@@ -1231,6 +1359,9 @@ function get_packages {
            if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
                file_to_parse="${file_to_parse} ${package_dir}/nova"
            fi
+            if [[ ! $file_to_parse =~ $package_dir/os-brick ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/os-brick"
+            fi
        elif [[ $service == g-* ]]; then
            if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
                file_to_parse="${file_to_parse} ${package_dir}/glance"
@@ -1363,7 +1494,7 @@ function uninstall_package {
    if is_ubuntu; then
        apt_get purge "$@"
    elif is_fedora; then
-        sudo ${YUM:-yum} remove -y "$@" ||:
+        sudo dnf remove -y "$@" ||:
    elif is_suse; then
        sudo zypper remove -y "$@" ||:
    else
@@ -1371,8 +1502,11 @@ function uninstall_package {
    fi
 }

-# Wrapper for ``yum`` to set proxy environment variables
-# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM``
+# Wrapper for ``dnf`` to set proxy environment variables
+# Uses globals ``OFFLINE``, ``*_proxy``
+# The name is kept for backwards compatability with external
+# callers, despite none of our supported platforms using yum
+# any more.
 # yum_install package [package ...]
 function yum_install {
    local result parse_yum_result
@@ -1380,44 +1514,8 @@ function yum_install {
    [[ "$OFFLINE" = "True" ]] && return

    time_start "yum_install"
-
-    # This is a bit tricky, because yum -y assumes missing or failed
-    # packages are OK (see [1]).  We want devstack to stop if we are
-    # installing missing packages.
-    #
-    # Thus we manually match on the output (stack.sh runs in a fixed
-    # locale, so lang shouldn't change).
-    #
-    # If yum returns !0, we echo the result as "YUM_FAILED" and return
-    # that from the awk (we're subverting -e with this trick).
-    # Otherwise we use awk to look for failure strings and return "2"
-    # to indicate a terminal failure.
-    #
-    # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567
-    parse_yum_result='              \
-        BEGIN { result=0 }          \
-        /^YUM_FAILED/ { result=$2 } \
-        /^No package/ { result=2 }  \
-        /^Failed:/    { result=2 }  \
-        //{ print }                 \
-        END { exit result }'
-    (sudo_with_proxies "${YUM:-yum}" install -y "$@" 2>&1 || echo YUM_FAILED $?) \
-        | awk "$parse_yum_result" && result=$? || result=$?
-
+    sudo_with_proxies dnf install -y "$@"
    time_stop "yum_install"
-
-    # if we return 1, then the wrapper functions will run an update
-    # and try installing the package again as a defense against bad
-    # mirrors.  This can hide failures, especially when we have
-    # packages that are in the "Failed:" section because their rpm
-    # install scripts failed to run correctly (in this case, the
-    # package looks installed, so when the retry happens we just think
-    # the package is OK, and incorrectly continue on).
-    if [ "$result" == 2 ]; then
-        die "Detected fatal package install failure"
-    fi
-
-    return "$result"
 }

 # zypper wrapper to set arguments correctly
@@ -1445,7 +1543,8 @@ function install_bindep {
    local pkgs

    if [[ ! -f $file ]]; then
-        die $LINENO "Can not find bindep file: $file"
+        warn $LINENO "Can not find bindep file: $file"
+        return
    fi

    # converting here makes it much easier to work with passing
@@ -1466,6 +1565,7 @@ function write_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local extra=""
    if [[ -n "$group" ]]; then
        extra="Group=$group"
@@ -1479,6 +1579,9 @@ function write_user_unit_file {
    iniset -sudo $unitfile "Service" "KillMode" "process"
    iniset -sudo $unitfile "Service" "TimeoutStopSec" "300"
    iniset -sudo $unitfile "Service" "ExecReload" "$KILL_PATH -HUP \$MAINPID"
+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1493,6 +1596,7 @@ function write_uwsgi_user_unit_file {
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"
    local unitfile="$SYSTEMD_DIR/$service"
    mkdir -p $SYSTEMD_DIR

@@ -1507,6 +1611,9 @@ function write_uwsgi_user_unit_file {
    iniset -sudo $unitfile "Service" "NotifyAccess" "all"
    iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"

+    if [[ -n "$env_vars" ]] ; then
+        iniset -sudo $unitfile "Service" "Environment" "$env_vars"
+    fi
    if [[ -n "$group" ]]; then
        iniset -sudo $unitfile "Service" "Group" "$group"
    fi
@@ -1554,10 +1661,14 @@ function _run_under_systemd {
    local systemd_service="devstack@$service.service"
    local group=$3
    local user=${4:-$STACK_USER}
+    if [[ -z "$user" ]]; then
+        user=$STACK_USER
+    fi
+    local env_vars="$5"
    if [[ "$command" =~ "uwsgi" ]] ; then
-        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    else
-        write_user_unit_file $systemd_service "$cmd" "$group" "$user"
+        write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
    fi

    $SYSTEMCTL enable $systemd_service
@@ -1578,18 +1689,20 @@ function is_running {
 # If the command includes shell metachatacters (;<>*) it must be run using a shell
 # If an optional group is provided sg will be used to run the
 # command as that group.
-# run_process service "command-line" [group] [user]
+# run_process service "command-line" [group] [user] [env_vars]
+# env_vars must be a space separated list of variable assigments, ie: "A=1 B=2"
 function run_process {
    local service=$1
    local command="$2"
    local group=$3
    local user=$4
+    local env_vars="$5"

    local name=$service

    time_start "run_process"
    if is_service_enabled $service; then
-        _run_under_systemd "$name" "$command" "$group" "$user"
+        _run_under_systemd "$name" "$command" "$group" "$user" "$env_vars"
    fi
    time_stop "run_process"
 }
@@ -1631,10 +1744,6 @@ function service_check {
 }


-function tail_log {
-    deprecated "With the removal of screen support, tail_log is deprecated and will be removed after Queens"
-}
-
 # Plugin Functions
 # =================

@@ -2075,11 +2184,7 @@ function _safe_permission_operation {
        return 0
    fi

-    if [[ $TRACK_DEPENDS = True ]]; then
-        sudo_cmd="env"
-    else
-        sudo_cmd="sudo"
-    fi
+    sudo_cmd="sudo"

    $xtrace
    $sudo_cmd $@
@@ -2441,6 +2546,13 @@ function time_totals {
    $xtrace
 }

+function clean_pyc_files {
+    # Clean up all *.pyc files
+    if [[ -n "$DEST" ]] && [[ -d "$DEST" ]]; then
+        sudo find $DEST -name "*.pyc" -delete
+    fi
+}
+
 # Restore xtrace
 $_XTRACE_FUNCTIONS_COMMON

@@ -0,0 +1,256 @@
+#!/bin/bash
+#
+# Symbolic asynchronous tasks for devstack
+#
+# Usage:
+#
+#  async_runfunc my_shell_func foo bar baz
+#
+#  ... do other stuff ...
+#
+#  async_wait my_shell_func
+#
+
+DEVSTACK_PARALLEL=$(trueorfalse True DEVSTACK_PARALLEL)
+_ASYNC_BG_TIME=0
+
+# Keep track of how much total time was spent in background tasks
+# Takes a job runtime in ms.
+function _async_incr_bg_time {
+    local elapsed_ms="$1"
+    _ASYNC_BG_TIME=$(($_ASYNC_BG_TIME + $elapsed_ms))
+}
+
+# Get the PID of a named future to wait on
+function async_pidof {
+    local name="$1"
+    local inifile="${DEST}/async/${name}.ini"
+
+    if [ -f "$inifile" ]; then
+        iniget $inifile job pid
+    else
+        echo 'UNKNOWN'
+        return 1
+    fi
+}
+
+# Log a message about a job. If the message contains "%command" then the
+# full command line of the job will be substituted in the output
+function async_log {
+    local name="$1"
+    shift
+    local message="$*"
+    local inifile=${DEST}/async/${name}.ini
+    local pid
+    local command
+
+    pid=$(iniget $inifile job pid)
+    command=$(iniget $inifile job command | tr '#' '-')
+    message=$(echo "$message" | sed "s#%command#$command#g")
+
+    echo "[$BASHPID Async ${name}:${pid}]: $message"
+}
+
+# Inner function that actually runs the requested task. We wrap it like this
+# just so we can emit a finish message as soon as the work is done, to make
+# it easier to find the tracking just before an error.
+function async_inner {
+    local name="$1"
+    local rc
+    local fifo="${DEST}/async/${name}.fifo"
+    shift
+    set -o xtrace
+    if $* >${DEST}/async/${name}.log 2>&1; then
+        rc=0
+        set +o xtrace
+        async_log "$name" "finished successfully"
+    else
+        rc=$?
+        set +o xtrace
+        async_log "$name" "FAILED with rc $rc"
+    fi
+    iniset ${DEST}/async/${name}.ini job end_time $(date "+%s%3N")
+    # Block on the fifo until we are signaled to exit by the main process
+    cat $fifo
+    return $rc
+}
+
+# Run something async. Takes a symbolic name and a list of arguments of
+# what to run. Ideally this would be rarely used and async_runfunc() would
+# be used everywhere for readability.
+#
+# This spawns the work in a background worker, records a "future" to be
+# collected by a later call to async_wait()
+function async_run {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    local name="$1"
+    shift
+    local inifile=${DEST}/async/${name}.ini
+    local fifo=${DEST}/async/${name}.fifo
+
+    touch $inifile
+    iniset $inifile job command "$*"
+    iniset $inifile job start_time $(date +%s%3N)
+
+    if [[ "$DEVSTACK_PARALLEL" = "True" ]]; then
+        mkfifo $fifo
+        async_inner $name $* &
+        iniset $inifile job pid $!
+        async_log "$name" "running: %command"
+        $xtrace
+    else
+        iniset $inifile job pid "self"
+        async_log "$name" "Running synchronously: %command"
+        $xtrace
+        $*
+        return $?
+    fi
+}
+
+# Shortcut for running a shell function async. Uses the function name as the
+# async name.
+function async_runfunc {
+    async_run $1 $*
+}
+
+# Dump some information to help debug a failed wait
+function async_wait_dump {
+    local failpid=$1
+
+    echo "=== Wait failure dump from $BASHPID ==="
+    echo "Processes:"
+    ps -f
+    echo "Waiting jobs:"
+    for name in $(ls ${DEST}/async/*.ini); do
+        echo "Job $name :"
+        cat "$name"
+    done
+    echo "Failed PID status:"
+    sudo cat /proc/$failpid/status
+    sudo cat /proc/$failpid/cmdline
+    echo "=== End wait failure dump ==="
+}
+
+# Wait for an async future to complete. May return immediately if already
+# complete, or of the future has already been waited on (avoid this). May
+# block until the future completes.
+function async_wait {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    local pid rc running inifile runtime fifo
+    rc=0
+    for name in $*; do
+        running=$(ls ${DEST}/async/*.ini 2>/dev/null | wc -l)
+        inifile="${DEST}/async/${name}.ini"
+        fifo="${DEST}/async/${name}.fifo"
+
+        if pid=$(async_pidof "$name"); then
+            async_log "$name" "Waiting for completion of %command" \
+                      "running on PID $pid ($running other jobs running)"
+            time_start async_wait
+            if [[ "$pid" != "self" ]]; then
+                # Signal the child to go ahead and exit since we are about to
+                # wait for it to collect its status.
+                async_log "$name" "Signaling child to exit"
+                echo WAKEUP > $fifo
+                async_log "$name" "Signaled"
+                # Do not actually call wait if we ran synchronously
+                if wait $pid; then
+                    rc=0
+                else
+                    rc=$?
+                fi
+                cat ${DEST}/async/${name}.log
+                rm -f $fifo
+            fi
+            time_stop async_wait
+            local start_time
+            local end_time
+            start_time=$(iniget $inifile job start_time)
+            end_time=$(iniget $inifile job end_time)
+            _async_incr_bg_time $(($end_time - $start_time))
+            runtime=$((($end_time - $start_time) / 1000))
+            async_log "$name" "finished %command with result" \
+                      "$rc in $runtime seconds"
+            rm -f $inifile
+            if [ $rc -ne 0 ]; then
+                async_wait_dump $pid
+                echo Stopping async wait due to error: $*
+                break
+            fi
+        else
+            # This could probably be removed - it is really just here
+            # to help notice if you wait for something by the wrong
+            # name, but it also shows up for things we didn't start
+            # because they were not enabled.
+            echo Not waiting for async task $name that we never started or \
+                 has already been waited for
+        fi
+    done
+
+    $xtrace
+    return $rc
+}
+
+# Check for uncollected futures and wait on them
+function async_cleanup {
+    local name
+
+    if [[ "$DEVSTACK_PARALLEL" != "True" ]]; then
+        return 0
+    fi
+
+    for inifile in $(find ${DEST}/async -name '*.ini'); do
+        name=$(basename $pidfile .ini)
+        echo "WARNING: uncollected async future $name"
+        async_wait $name || true
+    done
+}
+
+# Make sure our async dir is created and clean
+function async_init {
+    local async_dir=${DEST}/async
+
+    # Clean any residue if present from previous runs
+    rm -Rf $async_dir
+
+    # Make sure we have a state directory
+    mkdir -p $async_dir
+}
+
+function async_print_timing {
+    local bg_time_minus_wait
+    local elapsed_time
+    local serial_time
+    local speedup
+
+    if [[ "$DEVSTACK_PARALLEL" != "True" ]]; then
+        return 0
+    fi
+
+    # The logic here is: All the background task time would be
+    # serialized if we did not do them in the background. So we can
+    # add that to the elapsed time for the whole run. However, time we
+    # spend waiting for async things to finish adds to the elapsed
+    # time, but is time where we're not doing anything useful. Thus,
+    # we substract that from the would-be-serialized time.
+
+    bg_time_minus_wait=$((\
+            ($_ASYNC_BG_TIME - ${_TIME_TOTAL[async_wait]}) / 1000))
+    elapsed_time=$(($(date "+%s") - $_TIME_BEGIN))
+    serial_time=$(($elapsed_time + $bg_time_minus_wait))
+
+    echo
+    echo "================="
+    echo " Async summary"
+    echo "================="
+    echo " Time spent in the background minus waits: $bg_time_minus_wait sec"
+    echo " Elapsed time: $elapsed_time sec"
+    echo " Time if we did everything serially: $serial_time sec"
+    echo " Speedup: " $(echo | awk "{print $serial_time / $elapsed_time}")
+}
@@ -189,6 +189,9 @@ function iniset {
    local option=$3
    local value=$4

+    # Escape the ampersand character (&)
+    value=$(echo $value | sed -e 's/&/\\&/g')
+
    if [[ -z $section || -z $option ]]; then
        $xtrace
        return
@@ -21,6 +21,14 @@ set +o xtrace
 # project.  A null value installs to the system Python directories.
 declare -A -g PROJECT_VENV

+# Utility Functions
+# =================
+
+# Joins bash array of extras with commas as expected by other functions
+function join_extras {
+    local IFS=","
+    echo "$*"
+}

 # Python Functions
 # ================
@@ -54,7 +62,7 @@ function get_python_exec_prefix {
    $xtrace

    local PYTHON_PATH=/usr/local/bin
-    ( is_fedora && ! python3_enabled ) || is_suse && PYTHON_PATH=/usr/bin
+    is_suse && PYTHON_PATH=/usr/bin
    echo $PYTHON_PATH
 }

@@ -80,65 +88,13 @@ function pip_install_gr {
 function pip_install_gr_extras {
    local name=$1
    local extras=$2
-    local clean_name
-    clean_name=$(get_from_global_requirements $name)
-    pip_install $clean_name[$extras]
-}
-
-# python3_enabled_for() assumes the service(s) specified as arguments are
-# enabled for python 3 unless explicitly disabled. See python3_disabled_for().
-#
-# Multiple services specified as arguments are ``OR``'ed together; the test
-# is a short-circuit boolean, i.e it returns on the first match.
-#
-# python3_enabled_for dir [dir ...]
-function python3_enabled_for {
-    local xtrace
-    xtrace=$(set +o | grep xtrace)
-    set +o xtrace
-
-    local enabled=1
-    local dirs=$@
-    local dir
-    for dir in ${dirs}; do
-        if ! python3_disabled_for "${dir}"; then
-            enabled=0
-        fi
-    done
-
-    $xtrace
-    return $enabled
-}
-
-# python3_disabled_for() checks if the service(s) specified as arguments are
-# disabled by the user in ``DISABLED_PYTHON3_PACKAGES``.
-#
-# Multiple services specified as arguments are ``OR``'ed together; the test
-# is a short-circuit boolean, i.e it returns on the first match.
-#
-# Uses global ``DISABLED_PYTHON3_PACKAGES``
-# python3_disabled_for dir [dir ...]
-function python3_disabled_for {
-    local xtrace
-    xtrace=$(set +o | grep xtrace)
-    set +o xtrace
-
-    local enabled=1
-    local dirs=$@
-    local dir
-    for dir in ${dirs}; do
-        [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0
-    done
-
-    $xtrace
-    return $enabled
+    local version_constraints
+    version_constraints=$(get_version_constraints_from_global_requirements $name)
+    pip_install $name[$extras]$version_constraints
 }

 # enable_python3_package() -- no-op for backwards compatibility
 #
-# For example:
-#   enable_python3_package nova
-#
 # enable_python3_package dir [dir ...]
 function enable_python3_package {
    local xtrace
@@ -150,32 +106,22 @@ function enable_python3_package {
    $xtrace
 }

-# disable_python3_package() adds the services passed as argument to
-# the ``DISABLED_PYTHON3_PACKAGES`` list.
+# disable_python3_package() -- no-op for backwards compatibility
 #
-# For example:
-#   disable_python3_package swift
-#
-# Uses global ``DISABLED_PYTHON3_PACKAGES``
 # disable_python3_package dir [dir ...]
 function disable_python3_package {
    local xtrace
    xtrace=$(set +o | grep xtrace)
    set +o xtrace

-    local disabled_svcs="${DISABLED_PYTHON3_PACKAGES}"
-    local dir
-    for dir in $@; do
-        disabled_svcs+=",$dir"
-    done
-    DISABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$disabled_svcs")
+    echo "It is no longer possible to call disable_python3_package()."

    $xtrace
 }

 # Wrapper for ``pip install`` to set cache and proxy environment variables
 # Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
-# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
+# ``PIP_UPGRADE``, ``*_proxy``,
 # Usage:
 #  pip_install pip_arguments
 function pip_install {
@@ -219,74 +165,33 @@ function pip_install {
    # this works (for now...)
    local package_dir=${!#%\[*\]}

-    if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
-        # TRACK_DEPENDS=True installation creates a circular dependency when
-        # we attempt to install virtualenv into a virtualenv, so we must global
-        # that installation.
-        source $DEST/.venv/bin/activate
-        local cmd_pip=$DEST/.venv/bin/pip
+    if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
+        local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
        local sudo_pip="env"
    else
-        if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
-            local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
-            local sudo_pip="env"
-        else
-            local cmd_pip
-            cmd_pip=$(get_pip_command $PYTHON2_VERSION)
-            local sudo_pip="sudo -H"
-            if python3_enabled; then
-                # Special case some services that have experimental
-                # support for python3 in progress, but don't claim support
-                # in their classifier
-                echo "Check python version for : $package_dir"
-                if python3_disabled_for ${package_dir##*/}; then
-                    echo "Explicitly using $PYTHON2_VERSION version to install $package_dir based on DISABLED_PYTHON3_PACKAGES"
-                else
-                    # For everything that is not explicitly blacklisted with
-                    # DISABLED_PYTHON3_PACKAGES, assume it supports python3
-                    # and we will let pip sort out the install, regardless of
-                    # the package being local or remote.
-                    echo "Using $PYTHON3_VERSION version to install $package_dir based on default behavior"
-                    sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
-                    cmd_pip=$(get_pip_command $PYTHON3_VERSION)
-                fi
-            fi
-        fi
+        local cmd_pip="python$PYTHON3_VERSION -m pip"
+        # See
+        #  https://github.com/pypa/setuptools/issues/2232
+        #  http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
+        # this makes setuptools >=50 use the platform distutils.
+        # We only want to do this on global pip installs, not if
+        # installing in a virtualenv
+        local sudo_pip="sudo -H LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
+        echo "Using python $PYTHON3_VERSION to install $package_dir"
    fi

    cmd_pip="$cmd_pip install"
    # Always apply constraints
    cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"

-    # FIXME(dhellmann): Need to force multiple versions of pip for
-    # packages like setuptools?
-    local pip_version
-    pip_version=$(python -c "import pip; \
-                        print(pip.__version__.split('.')[0])")
-    if (( pip_version<6 )); then
-        die $LINENO "Currently installed pip version ${pip_version} does not" \
-            "meet minimum requirements (>=6)."
-    fi
-
    $xtrace

-    # Also install test requirements
-    local install_test_reqs=""
-    local test_req="${package_dir}/test-requirements.txt"
-    if [[ -e "$test_req" ]]; then
-        install_test_reqs="-r $test_req"
-    fi
-
-    # adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
-    # the same behaviour of setuptools before version 25.0.0.
-    # related issue: https://github.com/pypa/pip/issues/3874
    $sudo_pip \
        http_proxy="${http_proxy:-}" \
        https_proxy="${https_proxy:-}" \
        no_proxy="${no_proxy:-}" \
        PIP_FIND_LINKS=$PIP_FIND_LINKS \
-        SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
-        $cmd_pip $upgrade $install_test_reqs \
+        $cmd_pip $upgrade \
        $@
    result=$?

@@ -303,9 +208,8 @@ function pip_uninstall {
        local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
        local sudo_pip="env"
    else
-        local cmd_pip
-        cmd_pip=$(get_pip_command $PYTHON2_VERSION)
-        local sudo_pip="sudo -H"
+        local cmd_pip="python$PYTHON3_VERSION -m pip"
+        local sudo_pip="sudo -H LC_ALL=en_US.UTF-8"
    fi
    # don't error if we can't uninstall, it might not be there
    $sudo_pip $cmd_pip uninstall -y $name || /bin/true
@@ -323,6 +227,19 @@ function get_from_global_requirements {
    echo $required_pkg
 }

+# get only version constraints of a package from global requirements file
+# get_version_constraints_from_global_requirements <package>
+function get_version_constraints_from_global_requirements {
+    local package=$1
+    local required_pkg_version_constraint
+    # drop the package name from output (\K)
+    required_pkg_version_constraint=$(grep -i -h -o -P "^${package}\K.*" $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
+    if [[ $required_pkg_version_constraint == ""  ]]; then
+        die $LINENO "Can't find package $package in requirements"
+    fi
+    echo $required_pkg_version_constraint
+}
+
 # should we use this library from their git repo, or should we let it
 # get pulled in via pip dependencies.
 function use_library_from_git {
@@ -371,7 +288,7 @@ function setup_lib {
 #
 # use this for non namespaced libraries
 #
-# setup_dev_lib [-bindep] <name>
+# setup_dev_lib [-bindep] <name> [<extras>]
 function setup_dev_lib {
    local bindep
    if [[ $1 == -bindep* ]]; then
@@ -380,17 +297,8 @@ function setup_dev_lib {
    fi
    local name=$1
    local dir=${GITDIR[$name]}
-    if python3_enabled; then
-        # Turn off Python 3 mode and install the package again,
-        # forcing a Python 2 installation. This ensures that all libs
-        # being used for development are installed under both versions
-        # of Python.
-        echo "Installing $name again without Python 3 enabled"
-        USE_PYTHON3=False
-        setup_develop $bindep $dir
-        USE_PYTHON3=True
-    fi
-    setup_develop $bindep $dir
+    local extras=$2
+    setup_develop $bindep $dir $extras
 }

 # this should be used if you want to install globally, all libraries should
@@ -466,12 +374,13 @@ function _setup_package_with_constraints_edit {
    project_dir=$(cd $project_dir && pwd)

    if [ -n "$REQUIREMENTS_DIR" ]; then
-        # Constrain this package to this project directory from here on out.
+        # Remove this package from constraints before we install it.
+        # That way, later installs won't "downgrade" the install from
+        # source we are about to do.
        local name
        name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
        $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
-            $REQUIREMENTS_DIR/upper-constraints.txt -- $name \
-            "$flags file://$project_dir#egg=$name"
+            $REQUIREMENTS_DIR/upper-constraints.txt -- $name
    fi

    setup_package $bindep $project_dir "$flags" $extras
@@ -538,12 +447,15 @@ function setup_package {
 }

 # Report whether python 3 should be used
+# TODO(frickler): drop this once all legacy uses are removed
 function python3_enabled {
-    if [[ $USE_PYTHON3 == "True" ]]; then
-        return 0
-    else
-        return 1
-    fi
+    return 0
+}
+
+# Provide requested python version and sets PYTHON variable
+function install_python {
+    install_python3
+    export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null)
 }

 # Install python3 packages
@@ -552,6 +464,12 @@ function install_python3 {
        apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
    elif is_suse; then
        install_package python3-devel python3-dbm
+    elif is_fedora; then
+        if [ "$os_VENDOR" = "Fedora" ]; then
+            install_package python${PYTHON3_VERSION//.}
+        else
+            install_package python${PYTHON3_VERSION//.} python${PYTHON3_VERSION//.}-devel
+        fi
    fi
 }

@@ -27,6 +27,11 @@ set +o xtrace
 APACHE_USER=${APACHE_USER:-$STACK_USER}
 APACHE_GROUP=${APACHE_GROUP:-$(id -gn $APACHE_USER)}

+APACHE_LOCAL_HOST=$SERVICE_LOCAL_HOST
+if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
+    APACHE_LOCAL_HOST=[$APACHE_LOCAL_HOST]
+fi
+

 # Set up apache name and configuration directory
 # Note that APACHE_CONF_DIR is really more accurately apache's vhost
@@ -82,26 +87,43 @@ function install_apache_uwsgi {
        apxs="apxs"
    fi

-    # Ubuntu xenial is back level on uwsgi so the proxy doesn't
-    # actually work. Hence we have to build from source for now.
-    #
-    # Centos 7 actually has the module in epel, but there was a big
-    # push to disable epel by default. As such, compile from source
-    # there as well.
-
-    local dir
-    dir=$(mktemp -d)
-    pushd $dir
-    pip_install uwsgi
-    pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
-    local uwsgi
-    uwsgi=$(ls uwsgi*)
-    tar xvf $uwsgi
-    cd uwsgi*/apache2
-    sudo $apxs -i -c mod_proxy_uwsgi.c
-    popd
-    # delete the temp directory
-    sudo rm -rf $dir
+    if is_ubuntu; then
+        local pkg_list="uwsgi uwsgi-plugin-python3 libapache2-mod-proxy-uwsgi"
+        install_package ${pkg_list}
+    # NOTE(ianw) 2022-02-03 : Fedora 35 needs to skip this and fall
+    # into the install-from-source because the upstream packages
+    # didn't fix Python 3.10 compatibility before release.  Should be
+    # fixed in uwsgi 4.9.0; can remove this when packages available
+    # or we drop this release
+    elif is_fedora && ! [[ $DISTRO =~ f35 ]]; then
+        # Note httpd comes with mod_proxy_uwsgi and it is loaded by
+        # default; the mod_proxy_uwsgi package actually conflicts now.
+        # See:
+        #  https://bugzilla.redhat.com/show_bug.cgi?id=1574335
+        #
+        # Thus there is nothing else to do after this install
+        install_package uwsgi \
+                        uwsgi-plugin-python3
+    elif [[ $os_VENDOR =~ openSUSE ]]; then
+        install_package uwsgi \
+                        uwsgi-python3 \
+                        apache2-mod_uwsgi
+    else
+        # Compile uwsgi from source.
+        local dir
+        dir=$(mktemp -d)
+        pushd $dir
+        pip_install uwsgi
+        pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt
+        local uwsgi
+        uwsgi=$(ls uwsgi*)
+        tar xvf $uwsgi
+        cd uwsgi*/apache2
+        sudo $apxs -i -c mod_proxy_uwsgi.c
+        popd
+        # delete the temp directory
+        sudo rm -rf $dir
+    fi

    if is_ubuntu || is_suse ; then
        # we've got to enable proxy and proxy_uwsgi for this to work
@@ -121,17 +143,13 @@ function install_apache_wsgi {
    if is_ubuntu; then
        # Install apache2, which is NOPRIME'd
        install_package apache2
-        if python3_enabled; then
-            if is_package_installed libapache2-mod-wsgi; then
-                uninstall_package libapache2-mod-wsgi
-            fi
-            install_package libapache2-mod-wsgi-py3
-        else
-            install_package libapache2-mod-wsgi
+        if is_package_installed libapache2-mod-wsgi; then
+            uninstall_package libapache2-mod-wsgi
        fi
+        install_package libapache2-mod-wsgi-py3
    elif is_fedora; then
        sudo rm -f /etc/httpd/conf.d/000-*
-        install_package httpd mod_wsgi
+        install_package httpd python3-mod_wsgi
        # For consistency with Ubuntu, switch to the worker mpm, as
        # the default is event
        sudo sed -i '/mod_mpm_prefork.so/s/^/#/g' /etc/httpd/conf.modules.d/00-mpm.conf
@@ -265,7 +283,7 @@ function write_uwsgi_config {
    # configured after graceful shutdown
    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,python3
    # uwsgi recommends this to prevent thundering herd on accept.
    iniset "$file" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
@@ -285,7 +303,7 @@ function write_uwsgi_config {
        apache_conf=$(apache_site_config_for $name)
        iniset "$file" uwsgi socket "$socket"
        iniset "$file" uwsgi chmod-socket 666
-        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}/\" retry=0 " | sudo tee -a $apache_conf
+        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 " | sudo tee -a $apache_conf
        enable_apache_site $name
        restart_apache_server
    fi
@@ -310,7 +328,7 @@ function write_local_uwsgi_http_config {
    rm -rf $file
    iniset "$file" uwsgi wsgi-file "$wsgi"
    port=$(get_random_port)
-    iniset "$file" uwsgi http-socket "127.0.0.1:$port"
+    iniset "$file" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
    iniset "$file" uwsgi processes $API_WORKERS
    # This is running standalone
    iniset "$file" uwsgi master true
@@ -318,7 +336,7 @@ function write_local_uwsgi_http_config {
    iniset "$file" uwsgi die-on-term true
    iniset "$file" uwsgi exit-on-reload false
    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins python
+    iniset "$file" uwsgi plugins http,python3
    # uwsgi recommends this to prevent thundering herd on accept.
    iniset "$file" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
@@ -346,7 +364,26 @@ function write_local_uwsgi_http_config {
    apache_conf=$(apache_site_config_for $name)
    echo "KeepAlive Off" | sudo tee $apache_conf
    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
-    echo "ProxyPass \"${url}\" \"http://127.0.0.1:$port\" retry=0 " | sudo tee -a $apache_conf
+    echo "ProxyPass \"${url}\" \"http://$APACHE_LOCAL_HOST:$port\" retry=0 " | sudo tee -a $apache_conf
+    enable_apache_site $name
+    restart_apache_server
+}
+
+# Write a straight-through proxy for a service that runs locally and just needs
+# to be reachable via the main http proxy at $loc
+function write_local_proxy_http_config {
+    local name=$1
+    local url=$2
+    local loc=$3
+    local apache_conf
+    apache_conf=$(apache_site_config_for $name)
+
+    enable_apache_mod proxy
+    enable_apache_mod proxy_http
+
+    echo "KeepAlive Off" | sudo tee $apache_conf
+    echo "SetEnv proxy-sendchunked 1" | sudo tee -a $apache_conf
+    echo "ProxyPass \"${loc}\" \"$url\" retry=0 " | sudo tee -a $apache_conf
    enable_apache_site $name
    restart_apache_server
 }
@@ -31,6 +31,7 @@ set +o xtrace
 CINDER_DRIVER=${CINDER_DRIVER:-default}
 CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins
 CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends
+CINDER_BACKUPS=$TOP_DIR/lib/cinder_backups

 # grab plugin config if specified via cinder_driver
 if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
@@ -87,17 +88,44 @@ CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
 CINDER_VOLUME_CLEAR=${CINDER_VOLUME_CLEAR:-${CINDER_VOLUME_CLEAR_DEFAULT:-zero}}
 CINDER_VOLUME_CLEAR=$(echo ${CINDER_VOLUME_CLEAR} | tr '[:upper:]' '[:lower:]')

-# Centos7 and OpenSUSE switched to using LIO and that's all that's supported,
-# although the tgt bits are in EPEL and OpenSUSE we don't want that for CI
+# Default to lioadm
+CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-lioadm}
+
+VOLUME_TYPE_MULTIATTACH=${VOLUME_TYPE_MULTIATTACH:-multiattach}
+
+# EL and SUSE should only use lioadm
 if is_fedora || is_suse; then
-    CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-lioadm}
    if [[ ${CINDER_ISCSI_HELPER} != "lioadm" ]]; then
        die "lioadm is the only valid Cinder target_helper config on this platform"
    fi
-else
-    CINDER_ISCSI_HELPER=${CINDER_ISCSI_HELPER:-tgtadm}
 fi

+# When Cinder is used as a backend for Glance, it can be configured to clone
+# the volume containing image data directly in the backend instead of
+# transferring data from volume to volume.  Value is a comma separated list of
+# schemes (currently only 'file' and 'cinder' are supported).  The default
+# configuration in Cinder is empty (that is, do not use this feature).  NOTE:
+# to use this feature you must also enable GLANCE_SHOW_DIRECT_URL and/or
+# GLANCE_SHOW_MULTIPLE_LOCATIONS for glance-api.conf.
+CINDER_ALLOWED_DIRECT_URL_SCHEMES=${CINDER_ALLOWED_DIRECT_URL_SCHEMES:-}
+if [[ -n "$CINDER_ALLOWED_DIRECT_URL_SCHEMES" ]]; then
+    if [[ "${GLANCE_SHOW_DIRECT_URL:-False}" != "True" \
+            && "${GLANCE_SHOW_MULTIPLE_LOCATIONS:-False}" != "True" ]]; then
+        warn $LINENO "CINDER_ALLOWED_DIRECT_URL_SCHEMES is set, but neither \
+GLANCE_SHOW_DIRECT_URL nor GLANCE_SHOW_MULTIPLE_LOCATIONS is True"
+    fi
+fi
+
+# For backward compatibility
+# Before CINDER_BACKUP_DRIVER was introduced, ceph backup driver was configured
+# along with ceph backend driver.
+if [[ -z "${CINDER_BACKUP_DRIVER}" && "$CINDER_ENABLED_BACKENDS" =~ "ceph" ]]; then
+    CINDER_BACKUP_DRIVER=ceph
+fi
+
+# Supported backup drivers are in lib/cinder_backups
+CINDER_BACKUP_DRIVER=${CINDER_BACKUP_DRIVER:-swift}
+
 # Toggle for deploying Cinder under a wsgi server. Legacy mod_wsgi
 # reference should be cleaned up to more accurately refer to uwsgi.
 CINDER_USE_MOD_WSGI=${CINDER_USE_MOD_WSGI:-True}
@@ -113,6 +141,15 @@ if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
    done
 fi

+# Source the backup driver
+if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+    if [[ -r $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER ]]; then
+        source $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER
+    else
+        die "cinder backup driver $CINDER_BACKUP_DRIVER is not supported"
+    fi
+fi
+
 # Environment variables to configure the image-volume cache
 CINDER_IMG_CACHE_ENABLED=${CINDER_IMG_CACHE_ENABLED:-True}

@@ -125,6 +162,12 @@ CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
 # enable the cache for all cinder backends.
 CINDER_CACHE_ENABLED_FOR_BACKENDS=${CINDER_CACHE_ENABLED_FOR_BACKENDS:-$CINDER_ENABLED_BACKENDS}

+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the  Volume API policies to start checking the scope of token. by default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+CINDER_ENFORCE_SCOPE=$(trueorfalse False CINDER_ENFORCE_SCOPE)
+
 # Functions
 # ---------

@@ -189,6 +232,12 @@ function cleanup_cinder {
        done
    fi

+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type cleanup_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            cleanup_cinder_backup_$CINDER_BACKUP_DRIVER
+        fi
+    fi
+
    stop_process "c-api"
    remove_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI"
 }
@@ -235,6 +284,19 @@ function configure_cinder {
    fi
    iniset $CINDER_CONF key_manager backend cinder.keymgr.conf_key_mgr.ConfKeyManager
    iniset $CINDER_CONF key_manager fixed_key $(openssl rand -hex 16)
+    if [[ -n "$CINDER_ALLOWED_DIRECT_URL_SCHEMES" ]]; then
+        iniset $CINDER_CONF DEFAULT allowed_direct_url_schemes $CINDER_ALLOWED_DIRECT_URL_SCHEMES
+    fi
+
+    # set default quotas
+    iniset $CINDER_CONF DEFAULT quota_volumes ${CINDER_QUOTA_VOLUMES:-10}
+    iniset $CINDER_CONF DEFAULT quota_backups ${CINDER_QUOTA_BACKUPS:-10}
+    iniset $CINDER_CONF DEFAULT quota_snapshots ${CINDER_QUOTA_SNAPSHOTS:-10}
+
+    # Avoid RPC timeouts in slow CI and test environments by doubling the
+    # default response timeout set by RPC clients. See bug #1873234 for more
+    # details and example failures.
+    iniset $CINDER_CONF DEFAULT rpc_response_timeout 120

    if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
        local enabled_backends=""
@@ -250,9 +312,6 @@ function configure_cinder {
                default_name=$be_name
            fi
            enabled_backends+=$be_name,
-
-            iniset $CINDER_CONF $be_name volume_clear $CINDER_VOLUME_CLEAR
-
        done
        iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}
        if [[ -n "$default_name" ]]; then
@@ -261,13 +320,12 @@ function configure_cinder {
        configure_cinder_image_volume_cache
    fi

-    if is_service_enabled c-bak; then
-        # NOTE(mriedem): The default backup driver uses swift and if we're
-        # on a subnode we might not know if swift is enabled, but chances are
-        # good that it is on the controller so configure the backup service
-        # to use it. If we want to configure the backup service to use
-        # a non-swift driver, we'll likely need environment variables.
-        iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type configure_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            configure_cinder_backup_$CINDER_BACKUP_DRIVER
+        else
+            die "configure_cinder_backup_$CINDER_BACKUP_DRIVER doesn't exist in $CINDER_BACKUPS/$CINDER_BACKUP_DRIVER"
+        fi
    fi

    if is_service_enabled ceilometer; then
@@ -297,7 +355,9 @@ function configure_cinder {
    # Format logging
    setup_logging $CINDER_CONF $CINDER_USE_MOD_WSGI

-    write_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI" "/volume"
+    if is_service_enabled c-api; then
+        write_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI" "/volume"
+    fi

    if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
        configure_cinder_driver
@@ -311,10 +371,6 @@ function configure_cinder {
        iniset $CINDER_CONF DEFAULT glance_ca_certificates_file $SSL_BUNDLE_FILE
    fi

-    if [ "$GLANCE_V1_ENABLED" != "True" ]; then
-        iniset $CINDER_CONF DEFAULT glance_api_version 2
-    fi
-
    # Set nova credentials (used for os-assisted-snapshots)
    configure_keystone_authtoken_middleware $CINDER_CONF nova nova
    iniset $CINDER_CONF nova region_name "$REGION_NAME"
@@ -325,20 +381,33 @@ function configure_cinder {
    elif is_service_enabled etcd3; then
        iniset $CINDER_CONF coordination backend_url "etcd3+http://${SERVICE_HOST}:$ETCD_PORT"
    fi
+
+    if [[ "$CINDER_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $CINDER_CONF oslo_policy enforce_scope true
+        iniset $CINDER_CONF oslo_policy enforce_new_defaults true
+    fi
 }

 # create_cinder_accounts() - Set up common required cinder accounts

-# Tenant               User       Roles
+# Project              User       Roles
 # ------------------------------------------------------------------
-# service              cinder     admin        # if enabled
+# SERVICE_PROJECT_NAME cinder     service
+# SERVICE_PROJECT_NAME cinder     creator (if Barbican is enabled)

 # Migrated from keystone_data.sh
 function create_cinder_accounts {
    # Cinder
    if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

-        create_service_user "cinder"
+        local extra_role=""
+
+        # cinder needs the "creator" role in order to interact with barbican
+        if is_service_enabled barbican; then
+            extra_role=$(get_or_create_role "creator")
+        fi
+
+        create_service_user "cinder" $extra_role

        # block-storage is the official service type
        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
@@ -348,12 +417,6 @@ function create_cinder_accounts {
                "$REGION_NAME" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"

-            get_or_create_service "cinderv2" "volumev2" "Cinder Volume Service V2"
-            get_or_create_endpoint \
-                "volumev2" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(project_id)s"
-
            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
            get_or_create_endpoint \
                "volumev3" \
@@ -365,12 +428,6 @@ function create_cinder_accounts {
                "$REGION_NAME" \
                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"

-            get_or_create_service "cinderv2" "volumev2" "Cinder Volume Service V2"
-            get_or_create_endpoint \
-                "volumev2" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v2/\$(project_id)s"
-
            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
            get_or_create_endpoint \
                "volumev3" \
@@ -400,15 +457,17 @@ function init_cinder {
            be_type=${be%%:*}
            be_name=${be##*:}
            if type init_cinder_backend_${be_type} >/dev/null 2>&1; then
-                # Always init the default volume group for lvm.
-                if [[ "$be_type" == "lvm" ]]; then
-                    init_default_lvm_volume_group
-                fi
                init_cinder_backend_${be_type} ${be_name}
            fi
        done
    fi

+    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
+        if type init_cinder_backup_$CINDER_BACKUP_DRIVER >/dev/null 2>&1; then
+            init_cinder_backup_$CINDER_BACKUP_DRIVER
+        fi
+    fi
+
    mkdir -p $CINDER_STATE_PATH/volumes
 }

@@ -419,7 +478,7 @@ function install_cinder {
    if [[ "$CINDER_ISCSI_HELPER" == "tgtadm" ]]; then
        install_package tgt
    elif [[ "$CINDER_ISCSI_HELPER" == "lioadm" ]]; then
-        if [[ ${DISTRO} == "bionic" ]]; then
+        if is_ubuntu; then
            # TODO(frickler): Workaround for https://launchpad.net/bugs/1819819
            sudo mkdir -p /etc/target

@@ -492,7 +551,7 @@ function start_cinder {
                start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT
            fi
        else
-            run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF"
+            run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF"
            cinder_url=$service_protocol://$SERVICE_HOST/volume/v3
        fi
    fi
@@ -503,8 +562,13 @@ function start_cinder {
    fi

    run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
-    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
-    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
+    # Tune glibc for Python Services using single malloc arena for all threads
+    # and disabling dynamic thresholds to reduce memory usage when using native
+    # threads directly or via eventlet.tpool
+    # https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html
+    malloc_tuning="MALLOC_ARENA_MAX=1 MALLOC_MMAP_THRESHOLD_=131072 MALLOC_TRIM_THRESHOLD_=262144"
+    run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF" "" "" "$malloc_tuning"
+    run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF" "" "" "$malloc_tuning"

    # NOTE(jdg): For cinder, startup order matters.  To ensure that repor_capabilities is received
    # by the scheduler start the cinder-volume service last (or restart it) after the scheduler
@@ -519,6 +583,23 @@ function stop_cinder {
    stop_process c-vol
 }

+function create_one_type {
+    type_name=$1
+    property_key=$2
+    property_value=$3
+    # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
+    if is_service_enabled keystone; then
+        openstack --os-region-name="$REGION_NAME" volume type create --property $property_key="$property_value" $type_name
+    else
+        # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
+        # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
+        local cinder_url
+        cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create $type_name
+        OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key $type_name set $property_key="$property_value"
+    fi
+}
+
 # create_volume_types() - Create Cinder's configured volume types
 function create_volume_types {
    # Create volume types
@@ -526,18 +607,20 @@ function create_volume_types {
        local be be_name
        for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
            be_name=${be##*:}
-            # NOTE (e0ne): openstack client doesn't work with cinder in noauth mode
-            if is_service_enabled keystone; then
-                openstack --os-region-name="$REGION_NAME" volume type create --property volume_backend_name="${be_name}" ${be_name}
-            else
-                # TODO (e0ne): use openstack client once it will support cinder in noauth mode:
-                # https://bugs.launchpad.net/python-cinderclient/+bug/1755279
-                local cinder_url
-                cinder_url=$CINDER_SERVICE_PROTOCOL://$SERVICE_HOST:$CINDER_SERVICE_PORT/v3
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-create ${be_name}
-                OS_USER_ID=$OS_USERNAME OS_PROJECT_ID=$OS_PROJECT_NAME cinder --os-auth-type noauth --os-endpoint=$cinder_url type-key ${be_name} set volume_backend_name=${be_name}
-            fi
+            create_one_type $be_name "volume_backend_name" $be_name
        done
+
+        if [[ $ENABLE_VOLUME_MULTIATTACH == "True" ]]; then
+            create_one_type $VOLUME_TYPE_MULTIATTACH $VOLUME_TYPE_MULTIATTACH "<is> True"
+        fi
+
+        # Increase quota for the service project if glance is using cinder,
+        # since it's likely to occasionally go above the default 10 in parallel
+        # test execution.
+        if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then
+            openstack --os-region-name="$REGION_NAME" \
+                      quota set --volumes 50 "$SERVICE_PROJECT_NAME"
+        fi
    fi
 }

@@ -6,12 +6,6 @@
 # Enable with:
 #
 #   CINDER_ENABLED_BACKENDS+=,ceph:ceph
-#
-# Optional parameters:
-#   CINDER_BAK_CEPH_POOL=<pool-name>
-#   CINDER_BAK_CEPH_USER=<user>
-#   CINDER_BAK_CEPH_POOL_PG=<pg-num>
-#   CINDER_BAK_CEPH_POOL_PGP=<pgp-num>

 # Dependencies:
 #
@@ -29,11 +23,6 @@ set +o xtrace
 # Defaults
 # --------

-CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups}
-CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8}
-CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8}
-CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}
-

 # Entry Points
 # ------------
@@ -52,27 +41,6 @@ function configure_cinder_backend_ceph {
    iniset $CINDER_CONF $be_name rbd_flatten_volume_from_snapshot False
    iniset $CINDER_CONF $be_name rbd_max_clone_depth 5
    iniset $CINDER_CONF DEFAULT glance_api_version 2
-
-    if is_service_enabled c-bak; then
-        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
-        if [ "$REMOTE_CEPH" = "False" ]; then
-            # Configure Cinder backup service options, ceph pool, ceph user and ceph key
-            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} size ${CEPH_REPLICAS}
-            if [[ $CEPH_REPLICAS -ne 1 ]]; then
-                sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
-            fi
-        fi
-        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
-
-        iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
-        iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
-        iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL"
-        iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER"
-        iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0
-        iniset $CINDER_CONF DEFAULT backup_ceph_stripe_count 0
-        iniset $CINDER_CONF DEFAULT restore_discard_excess_bytes True
-    fi
 }

 # Restore xtrace
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# lib/cinder_backends/ceph_iscsi
+# Configure the ceph_iscsi backend
+
+# Enable with:
+#
+#   CINDER_ENABLED_BACKENDS+=,ceph_iscsi:ceph_iscsi
+#
+# Optional paramteters:
+#   CEPH_ISCSI_API_URL=<url to the rbd-target-api service>
+#
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# configure_ceph_backend_ceph_iscsi - called from configure_cinder()
+
+
+# Save trace setting
+_XTRACE_CINDER_CEPH_ISCSI=$(set +o | grep xtrace)
+set +o xtrace
+
+# Entry Points
+# ------------
+
+# configure_cinder_backend_ceph_iscsi - Set config files, create data dirs, etc
+# configure_cinder_backend_ceph_iscsi $name
+function configure_cinder_backend_ceph_iscsi {
+    local be_name=$1
+
+    CEPH_ISCSI_API_URL=${CEPH_ISCSI_API_URL:-http://$CEPH_ISCSI_API_HOST:$CEPH_ISCSI_API_PORT}
+
+    iniset $CINDER_CONF $be_name volume_backend_name $be_name
+    iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.ceph.rbd_iscsi.RBDISCSIDriver"
+    iniset $CINDER_CONF $be_name rbd_ceph_conf "$CEPH_CONF_FILE"
+    iniset $CINDER_CONF $be_name rbd_pool "$CINDER_CEPH_POOL"
+    iniset $CINDER_CONF $be_name rbd_user "$CINDER_CEPH_USER"
+    iniset $CINDER_CONF $be_name rbd_iscsi_api_user "$CEPH_ISCSI_API_USER"
+    iniset $CINDER_CONF $be_name rbd_iscsi_api_password "$CEPH_ISCSI_API_PASSWORD"
+    iniset $CINDER_CONF $be_name rbd_iscsi_api_url "$CEPH_ISCSI_API_URL"
+    iniset $CINDER_CONF $be_name rbd_iscsi_target_iqn "$CEPH_ISCSI_TARGET_IQN"
+    iniset $CINDER_CONF $be_name rbd_flatten_volume_from_snapshot False
+    iniset $CINDER_CONF $be_name rbd_max_clone_depth 5
+    iniset $CINDER_CONF DEFAULT glance_api_version 2
+
+    pip_install rbd-iscsi-client
+}
+
+# Restore xtrace
+$_XTRACE_CINDER_CEPH_ISCSI
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -52,7 +52,7 @@ function configure_cinder_backend_lvm {
    iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
    iniset $CINDER_CONF $be_name target_helper "$CINDER_ISCSI_HELPER"
    iniset $CINDER_CONF $be_name lvm_type "$CINDER_LVM_TYPE"
-
+    iniset $CINDER_CONF $be_name volume_clear "$CINDER_VOLUME_CLEAR"
 }

 # init_cinder_backend_lvm - Initialize volume group
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# lib/cinder_backups/ceph
+# Configure the ceph backup driver
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=ceph
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_CEPH=$(set +o | grep xtrace)
+set +o xtrace
+
+# Defaults
+# --------
+
+CINDER_BAK_CEPH_POOL=${CINDER_BAK_CEPH_POOL:-backups}
+CINDER_BAK_CEPH_POOL_PG=${CINDER_BAK_CEPH_POOL_PG:-8}
+CINDER_BAK_CEPH_POOL_PGP=${CINDER_BAK_CEPH_POOL_PGP:-8}
+CINDER_BAK_CEPH_USER=${CINDER_BAK_CEPH_USER:-cinder-bak}
+
+
+function configure_cinder_backup_ceph {
+    # Execute this part only when cephadm is not used
+    if [[ "$CEPHADM_DEPLOY" = "False" ]]; then
+        sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_BAK_CEPH_POOL} ${CINDER_BAK_CEPH_POOL_PG} ${CINDER_BAK_CEPH_POOL_PGP}
+        if [[ "$REMOTE_CEPH" = "False" && "$CEPH_REPLICAS" -ne 1 ]]; then
+            sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_BAK_CEPH_POOL} crush_ruleset ${RULE_ID}
+        fi
+        sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_BAK_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_BAK_CEPH_POOL}, allow rwx pool=${CINDER_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+        sudo chown $(whoami):$(whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_BAK_CEPH_USER}.keyring
+    fi
+
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.ceph.CephBackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_ceph_conf "$CEPH_CONF_FILE"
+    iniset $CINDER_CONF DEFAULT backup_ceph_pool "$CINDER_BAK_CEPH_POOL"
+    iniset $CINDER_CONF DEFAULT backup_ceph_user "$CINDER_BAK_CEPH_USER"
+    iniset $CINDER_CONF DEFAULT backup_ceph_stripe_unit 0
+    iniset $CINDER_CONF DEFAULT backup_ceph_stripe_count 0
+    iniset $CINDER_CONF DEFAULT restore_discard_excess_bytes True
+}
+
+# init_cinder_backup_ceph: nothing to do
+# cleanup_cinder_backup_ceph: nothing to do
+
+# Restore xtrace
+$_XTRACE_CINDER_CEPH
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# lib/cinder_backups/s3_swift
+# Configure the s3 backup driver with swift s3api
+#
+# TODO: create lib/cinder_backup/s3 for external s3 compatible storage
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=s3_swift
+#   enable_service s3api s-proxy s-object s-container s-account
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_S3_SWIFT=$(set +o | grep xtrace)
+set +o xtrace
+
+function configure_cinder_backup_s3_swift {
+    # This configuration requires swift and s3api. If we're
+    # on a subnode we might not know if they are enabled
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.s3.S3BackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_s3_endpoint_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$S3_SERVICE_PORT"
+}
+
+function init_cinder_backup_s3_swift {
+    openstack ec2 credential create
+    iniset $CINDER_CONF DEFAULT backup_s3_store_access_key "$(openstack ec2 credential list -c Access -f value)"
+    iniset $CINDER_CONF DEFAULT backup_s3_store_secret_key "$(openstack ec2 credential list -c Secret -f value)"
+    if is_service_enabled tls-proxy; then
+        iniset $CINDER_CONF DEFAULT backup_s3_ca_cert_file "$SSL_BUNDLE_FILE"
+    fi
+}
+
+# cleanup_cinder_backup_s3_swift: nothing to do
+
+# Restore xtrace
+$_XTRACE_CINDER_S3_SWIFT
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# lib/cinder_backups/swift
+# Configure the swift backup driver
+
+# Enable with:
+#
+#   CINDER_BACKUP_DRIVER=swift
+
+# Dependencies:
+#
+# - ``functions`` file
+# - ``cinder`` configurations
+
+# Save trace setting
+_XTRACE_CINDER_SWIFT=$(set +o | grep xtrace)
+set +o xtrace
+
+
+function configure_cinder_backup_swift {
+    # NOTE(mriedem): The default backup driver uses swift and if we're
+    # on a subnode we might not know if swift is enabled, but chances are
+    # good that it is on the controller so configure the backup service
+    # to use it.
+    iniset $CINDER_CONF DEFAULT backup_driver "cinder.backup.drivers.swift.SwiftBackupDriver"
+    iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_"
+    if is_service_enabled tls-proxy; then
+        iniset $CINDER_CONF DEFAULT backup_swift_ca_cert_file $SSL_BUNDLE_FILE
+    fi
+}
+
+# init_cinder_backup_swift: nothing to do
+# cleanup_cinder_backup_swift: nothing to do
+
+
+# Restore xtrace
+$_XTRACE_CINDER_SWIFT
+
+# Local variables:
+# mode: shell-script
+# End:
@@ -1,46 +0,0 @@
-#!/bin/bash
-#
-# lib/cinder_plugins/XenAPINFS
-# Configure the XenAPINFS driver
-
-# Enable with:
-#
-#   CINDER_DRIVER=XenAPINFS
-
-# Dependencies:
-#
-# - ``functions`` file
-# - ``cinder`` configurations
-
-# configure_cinder_driver - make configuration changes, including those to other services
-
-# Save trace setting
-_XTRACE_CINDER_XENAPINFS=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-# Set up default directories
-
-
-# Entry Points
-# ------------
-
-# configure_cinder_driver - Set config files, create data dirs, etc
-function configure_cinder_driver {
-    iniset $CINDER_CONF DEFAULT volume_driver "cinder.volume.drivers.xenapi.sm.XenAPINFSDriver"
-    iniset $CINDER_CONF DEFAULT xenapi_connection_url "$CINDER_XENAPI_CONNECTION_URL"
-    iniset $CINDER_CONF DEFAULT xenapi_connection_username "$CINDER_XENAPI_CONNECTION_USERNAME"
-    iniset $CINDER_CONF DEFAULT xenapi_connection_password "$CINDER_XENAPI_CONNECTION_PASSWORD"
-    iniset $CINDER_CONF DEFAULT xenapi_nfs_server "$CINDER_XENAPI_NFS_SERVER"
-    iniset $CINDER_CONF DEFAULT xenapi_nfs_serverpath "$CINDER_XENAPI_NFS_SERVERPATH"
-}
-
-# Restore xtrace
-$_XTRACE_CINDER_XENAPINFS
-
-# Local variables:
-# mode: shell-script
-# End:
@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# lib/cinder_plugins/sheepdog
-# Configure the sheepdog driver
-
-# Enable with:
-#
-#   CINDER_DRIVER=sheepdog
-
-# Dependencies:
-#
-# - ``functions`` file
-# - ``cinder`` configurations
-
-# configure_cinder_driver - make configuration changes, including those to other services
-
-# Save trace setting
-_XTRACE_CINDER_SHEEPDOG=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-# Set up default directories
-
-
-# Entry Points
-# ------------
-
-# configure_cinder_driver - Set config files, create data dirs, etc
-function configure_cinder_driver {
-    iniset $CINDER_CONF DEFAULT volume_driver "cinder.volume.drivers.sheepdog.SheepdogDriver"
-}
-
-# Restore xtrace
-$_XTRACE_CINDER_SHEEPDOG
-
-# Local variables:
-# mode: shell-script
-# End:
@@ -89,6 +89,10 @@ function initialize_database_backends {
        DATABASE_PASSWORD=$MYSQL_PASSWORD
    fi

+    return 0
+}
+
+function define_database_baseurl {
    # We configure Nova, Horizon, Glance and Keystone to use MySQL as their
    # database server.  While they share a single server, each has their own
    # database and tables.
@@ -100,8 +104,6 @@ function initialize_database_backends {

    # NOTE: Don't specify ``/db`` in this string so we can use it for multiple services
    BASE_SQL_CONN=${BASE_SQL_CONN:-$(get_database_type_$DATABASE_TYPE)://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOST}
-
-    return 0
 }

 # Recreate a given database
@@ -12,18 +12,23 @@ _XTRACE_DB_MYSQL=$(set +o | grep xtrace)
 set +o xtrace

 MYSQL_DRIVER=${MYSQL_DRIVER:-PyMySQL}
+INSTALL_DATABASE_SERVER_PACKAGES=$(trueorfalse True INSTALL_DATABASE_SERVER_PACKAGES)

 register_database mysql

-MYSQL_SERVICE_NAME=mysql
-if is_fedora && ! is_oraclelinux; then
-    MYSQL_SERVICE_NAME=mariadb
-elif is_suse && systemctl list-unit-files | grep -q 'mariadb\.service'; then
-    # Older mariadb packages on SLES 12 provided mysql.service.  The
-    # newer ones on SLES 12 and 15 use mariadb.service; they also
-    # provide a mysql.service symlink for backwards-compatibility, but
-    # let's not rely on that.
-    MYSQL_SERVICE_NAME=mariadb
+if [[ -z "$MYSQL_SERVICE_NAME" ]]; then
+    MYSQL_SERVICE_NAME=mysql
+    if is_fedora && ! is_oraclelinux; then
+        MYSQL_SERVICE_NAME=mariadb
+    elif is_suse && systemctl list-unit-files | grep -q 'mariadb\.service'; then
+        # Older mariadb packages on SLES 12 provided mysql.service.  The
+        # newer ones on SLES 12 and 15 use mariadb.service; they also
+        # provide a mysql.service symlink for backwards-compatibility, but
+        # let's not rely on that.
+        MYSQL_SERVICE_NAME=mariadb
+    elif [[ "$DISTRO" == "bullseye" ]]; then
+        MYSQL_SERVICE_NAME=mariadb
+    fi
 fi

 # Functions
@@ -81,10 +86,16 @@ function configure_database_mysql {
        exit_distro_not_supported "mysql configuration"
    fi

-    # Start mysql-server
+    # Change bind-address from localhost (127.0.0.1) to any (::)
+    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"
+
+    # (Re)Start mysql-server
    if is_fedora || is_suse; then
        # service is not started by default
        start_service $MYSQL_SERVICE_NAME
+    elif is_ubuntu; then
+        # required since bind-address could have changed above
+        restart_service $MYSQL_SERVICE_NAME
    fi

    # Set the root password - only works the first time. For Ubuntu, we already
@@ -92,19 +103,32 @@ function configure_database_mysql {
    # because the package might have been installed already.
    sudo mysqladmin -u root password $DATABASE_PASSWORD || true

+    # In case of Mariadb, giving hostname in arguments causes permission
+    # problems as it expects connection through socket
+    if is_ubuntu && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
+        local cmd_args="-uroot -p$DATABASE_PASSWORD "
+    else
+        local cmd_args="-uroot -p$DATABASE_PASSWORD -h$SERVICE_LOCAL_HOST "
+    fi
+
+    # In mariadb e.g. on Ubuntu socket plugin is used for authentication
+    # as root so it works only as sudo. To restore old "mysql like" behaviour,
+    # we need to change auth plugin for root user
+    if is_ubuntu && [[ "$DISTRO" != "bullseye" ]] && [ "$MYSQL_SERVICE_NAME" == "mariadb" ]; then
+        sudo mysql $cmd_args -e "UPDATE mysql.user SET plugin='' WHERE user='$DATABASE_USER' AND host='localhost';"
+        sudo mysql $cmd_args -e "FLUSH PRIVILEGES;"
+    fi
+    # Create DB user if it does not already exist
+    sudo mysql $cmd_args -e "CREATE USER IF NOT EXISTS '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
    # Update the DB to give user '$DATABASE_USER'@'%' full control of the all databases:
-    sudo mysql -uroot -p$DATABASE_PASSWORD -h127.0.0.1 -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%' identified by '$DATABASE_PASSWORD';"
+    sudo mysql $cmd_args -e "GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'%';"

    # Now update ``my.cnf`` for some local needs and restart the mysql service

-    # Change bind-address from localhost (127.0.0.1) to any (::) and
-    # set default db type to InnoDB
-    iniset -sudo $my_conf mysqld bind-address "$(ipv6_unquote $SERVICE_LISTEN_ADDRESS)"
+    # Set default db type to InnoDB
    iniset -sudo $my_conf mysqld sql_mode TRADITIONAL
    iniset -sudo $my_conf mysqld default-storage-engine InnoDB
    iniset -sudo $my_conf mysqld max_connections 1024
-    iniset -sudo $my_conf mysqld query_cache_type OFF
-    iniset -sudo $my_conf mysqld query_cache_size 0

    if [[ "$DATABASE_QUERY_LOGGING" == "True" ]]; then
        echo_summary "Enabling MySQL query logging"
@@ -126,6 +150,29 @@ function configure_database_mysql {
        iniset -sudo $my_conf mysqld log-queries-not-using-indexes 1
    fi

+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        echo "enabling MySQL performance counting"
+
+        # Install our sqlalchemy plugin
+        pip_install ${TOP_DIR}/tools/dbcounter
+
+        # Create our stats database for accounting
+        recreate_database stats
+        mysql -u $DATABASE_USER -p$DATABASE_PASSWORD -h $MYSQL_HOST -e \
+              "CREATE TABLE queries (db VARCHAR(32), op VARCHAR(32),
+                count INT, PRIMARY KEY (db, op)) ENGINE MEMORY" stats
+    fi
+
+    if [[ "$MYSQL_REDUCE_MEMORY" == "True" ]]; then
+        iniset -sudo $my_conf mysqld read_buffer_size 64K
+        iniset -sudo $my_conf mysqld innodb_buffer_pool_size 16M
+        iniset -sudo $my_conf mysqld thread_stack 192K
+        iniset -sudo $my_conf mysqld thread_cache_size 8
+        iniset -sudo $my_conf mysqld tmp_table_size 8M
+        iniset -sudo $my_conf mysqld sort_buffer_size 8M
+        iniset -sudo $my_conf mysqld max_allowed_packet 8M
+    fi
+
    restart_service $MYSQL_SERVICE_NAME
 }

@@ -148,20 +195,28 @@ MYSQL_PRESEED
 [client]
 user=$DATABASE_USER
 password=$DATABASE_PASSWORD
-host=$MYSQL_HOST
 EOF
+
+        if ! is_ubuntu || [ "$MYSQL_SERVICE_NAME" != "mariadb" ]; then
+            echo "host=$MYSQL_HOST" >> $HOME/.my.cnf
+        fi
        chmod 0600 $HOME/.my.cnf
    fi
    # Install mysql-server
-    if is_oraclelinux; then
-        install_package mysql-community-server
-    elif is_fedora || is_suse; then
-        install_package mariadb-server
-        sudo systemctl enable $MYSQL_SERVICE_NAME
-    elif is_ubuntu; then
-        install_package mysql-server
-    else
-        exit_distro_not_supported "mysql installation"
+    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
+        if is_oraclelinux; then
+            install_package mysql-community-server
+        elif is_fedora; then
+            install_package mariadb-server mariadb-devel mariadb
+            sudo systemctl enable $MYSQL_SERVICE_NAME
+        elif is_suse; then
+            install_package mariadb-server
+            sudo systemctl enable $MYSQL_SERVICE_NAME
+        elif is_ubuntu; then
+            install_package $MYSQL_SERVICE_NAME-server
+        else
+            exit_distro_not_supported "mysql installation"
+        fi
    fi
 }

@@ -177,7 +232,17 @@ function install_database_python_mysql {

 function database_connection_url_mysql {
    local db=$1
-    echo "$BASE_SQL_CONN/$db?charset=utf8"
+    local plugin
+
+    # NOTE(danms): We don't enable perf on subnodes yet because the
+    # plugin is not installed there
+    if [[ "$MYSQL_GATHER_PERFORMANCE" == "True" ]]; then
+        if is_service_enabled mysql; then
+            plugin="&plugin=dbcounter"
+        fi
+    fi
+
+    echo "$BASE_SQL_CONN/$db?charset=utf8$plugin"
 }


@@ -13,7 +13,7 @@ set +o xtrace


 MAX_DB_CONNECTIONS=${MAX_DB_CONNECTIONS:-200}
-
+INSTALL_DATABASE_SERVER_PACKAGES=$(trueorfalse True INSTALL_DATABASE_SERVER_PACKAGES)

 register_database postgresql

@@ -53,7 +53,8 @@ function configure_database_postgresql {
        pg_hba=/var/lib/pgsql/data/pg_hba.conf
        pg_conf=/var/lib/pgsql/data/postgresql.conf
        if ! sudo [ -e $pg_hba ]; then
-            sudo postgresql-setup initdb
+            localectl set-locale LANG=en_US.UTF-8
+            sudo -E postgresql-setup initdb
        fi
    elif is_ubuntu; then
        version=`psql --version | cut -d ' ' -f3 | cut -d. -f1-2`
@@ -95,7 +96,6 @@ function configure_database_postgresql {

 function install_database_postgresql {
    echo_summary "Installing postgresql"
-    deprecated "Use of postgresql in devstack is deprecated, and will be removed during the Pike cycle"
    local pgpass=$HOME/.pgpass
    if [[ ! -e $pgpass ]]; then
        cat <<EOF > $pgpass
@@ -105,15 +105,17 @@ EOF
    else
        sed -i "s/:root:\w\+/:root:$DATABASE_PASSWORD/" $pgpass
    fi
-    if is_ubuntu; then
-        install_package postgresql
-    elif is_fedora || is_suse; then
-        install_package postgresql-server
-        if is_fedora; then
-            sudo systemctl enable postgresql
+    if [[ "$INSTALL_DATABASE_SERVER_PACKAGES" == "True" ]]; then
+        if is_ubuntu; then
+            install_package postgresql
+        elif is_fedora || is_suse; then
+            install_package postgresql-server langpacks-en glibc-langpack-en
+            if is_fedora; then
+                sudo systemctl enable postgresql
+            fi
+        else
+            exit_distro_not_supported "postgresql installation"
        fi
-    else
-        exit_distro_not_supported "postgresql installation"
    fi
 }

@@ -9,6 +9,7 @@

 # ``stack.sh`` calls the entry points in this order:
 #
+# - install_dstat
 # - start_dstat
 # - stop_dstat

@@ -16,6 +17,14 @@
 _XTRACE_DSTAT=$(set +o | grep xtrace)
 set +o xtrace

+# install_dstat() - Install prerequisites for dstat services
+function install_dstat {
+    if is_service_enabled memory_tracker; then
+        # Install python libraries required by tools/mlock_report.py
+        pip_install_gr psutil
+    fi
+}
+
 # start_dstat() - Start running processes
 function start_dstat {
    # A better kind of sysstat, with the top process per time slice
@@ -26,10 +35,10 @@ function start_dstat {
    # to your localrc
    run_process memory_tracker "$TOP_DIR/tools/memory_tracker.sh" "" "root"

-    # remove support for the old name when it's no longer used (sometime in Queens)
+    # TODO(jh): Fail when using the old service name otherwise consumers might
+    # never notice that is has been removed.
    if is_service_enabled peakmem_tracker; then
-        deprecated "Use of peakmem_tracker in devstack is deprecated, use memory_tracker instead"
-        run_process peakmem_tracker "$TOP_DIR/tools/memory_tracker.sh" "" "root"
+        die $LINENO "The peakmem_tracker service has been removed, use memory_tracker instead"
    fi
 }

@@ -41,22 +41,77 @@ else
    GLANCE_BIN_DIR=$(get_python_exec_prefix)
 fi

+# Cinder for Glance
+USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE)
+# GLANCE_CINDER_DEFAULT_BACKEND should be one of the values
+# from CINDER_ENABLED_BACKENDS
+GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
+GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
+# NOTE (abhishekk): For opensuse data files are stored in different directory
+if is_opensuse; then
+    GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/etc/glance
+fi
+# When Cinder is used as a glance store, you can optionally configure cinder to
+# optimize bootable volume creation by allowing volumes to be cloned directly
+# in the backend instead of transferring data via Glance.  To use this feature,
+# set CINDER_ALLOWED_DIRECT_URL_SCHEMES for cinder.conf and enable
+# GLANCE_SHOW_DIRECT_URL and/or GLANCE_SHOW_MULTIPLE_LOCATIONS for Glance.  The
+# default value for both of these is False, because for some backends they
+# present a grave security risk (though not for Cinder, because all that's
+# exposed is the volume_id where the image data is stored.)  See OSSN-0065 for
+# more information: https://wiki.openstack.org/wiki/OSSN/OSSN-0065
+GLANCE_SHOW_DIRECT_URL=$(trueorfalse False GLANCE_SHOW_DIRECT_URL)
+GLANCE_SHOW_MULTIPLE_LOCATIONS=$(trueorfalse False GLANCE_SHOW_MULTIPLE_LOCATIONS)
+
+# Glance multi-store configuration
+# Boolean flag to enable multiple store configuration for glance
+GLANCE_ENABLE_MULTIPLE_STORES=$(trueorfalse False GLANCE_ENABLE_MULTIPLE_STORES)
+
+# Comma separated list for configuring multiple file stores of glance,
+# for example; GLANCE_MULTIPLE_FILE_STORES = fast,cheap,slow
+GLANCE_MULTIPLE_FILE_STORES=${GLANCE_MULTIPLE_FILE_STORES:-fast}
+
+# Default store/backend for glance, must be one of the store specified
+# in GLANCE_MULTIPLE_FILE_STORES option.
+GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:-fast}
+
 GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache}
+
+# Full Glance functionality requires running in standalone mode. If we are
+# not in uwsgi mode, then we are standalone, otherwise allow separate control.
+if [[ "$WSGI_MODE" != "uwsgi" ]]; then
+    GLANCE_STANDALONE=True
+fi
+GLANCE_STANDALONE=${GLANCE_STANDALONE:-False}
+
+# File path for each store specified in GLANCE_MULTIPLE_FILE_STORES, the store
+# identifier will be appended to this path at runtime. If GLANCE_MULTIPLE_FILE_STORES
+# has fast,cheap specified then filepath will be generated like $DATA_DIR/glance/fast
+# and $DATA_DIR/glance/cheap.
+GLANCE_MULTISTORE_FILE_IMAGE_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/glance}
 GLANCE_IMAGE_DIR=${GLANCE_IMAGE_DIR:=$DATA_DIR/glance/images}
+GLANCE_NFS_MOUNTPOINT=$GLANCE_IMAGE_DIR/mnt
 GLANCE_LOCK_DIR=${GLANCE_LOCK_DIR:=$DATA_DIR/glance/locks}
+GLANCE_STAGING_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_staging_store}
+GLANCE_TASKS_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_tasks_store}
+
+GLANCE_USE_IMPORT_WORKFLOW=$(trueorfalse False GLANCE_USE_IMPORT_WORKFLOW)
+GLANCE_ENABLE_QUOTAS=$(trueorfalse True GLANCE_ENABLE_QUOTAS)
+
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the Image API policies to start checking the scope of token. By Default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+GLANCE_ENFORCE_SCOPE=$(trueorfalse False GLANCE_ENFORCE_SCOPE)

 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
-GLANCE_REGISTRY_CONF=$GLANCE_CONF_DIR/glance-registry.conf
 GLANCE_API_CONF=$GLANCE_CONF_DIR/glance-api.conf
-GLANCE_REGISTRY_PASTE_INI=$GLANCE_CONF_DIR/glance-registry-paste.ini
 GLANCE_API_PASTE_INI=$GLANCE_CONF_DIR/glance-api-paste.ini
 GLANCE_CACHE_CONF=$GLANCE_CONF_DIR/glance-cache.conf
-GLANCE_POLICY_JSON=$GLANCE_CONF_DIR/policy.json
 GLANCE_SCHEMA_JSON=$GLANCE_CONF_DIR/schema-image.json
 GLANCE_SWIFT_STORE_CONF=$GLANCE_CONF_DIR/glance-swift-store.conf
 GLANCE_IMAGE_IMPORT_CONF=$GLANCE_CONF_DIR/glance-image-import.conf
-GLANCE_V1_ENABLED=${GLANCE_V1_ENABLED:-False}

 if is_service_enabled tls-proxy; then
    GLANCE_SERVICE_PROTOCOL="https"
@@ -69,10 +124,12 @@ GLANCE_SERVICE_PORT=${GLANCE_SERVICE_PORT:-9292}
 GLANCE_SERVICE_PORT_INT=${GLANCE_SERVICE_PORT_INT:-19292}
 GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT}
 GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
-GLANCE_REGISTRY_PORT=${GLANCE_REGISTRY_PORT:-9191}
-GLANCE_REGISTRY_PORT_INT=${GLANCE_REGISTRY_PORT_INT:-19191}
 GLANCE_UWSGI=$GLANCE_BIN_DIR/glance-wsgi-api
 GLANCE_UWSGI_CONF=$GLANCE_CONF_DIR/glance-uwsgi.ini
+
+# Glance default limit for Devstack
+GLANCE_LIMIT_IMAGE_SIZE_TOTAL=${GLANCE_LIMIT_IMAGE_SIZE_TOTAL:-1000}
+
 # If wsgi mode is uwsgi run glance under uwsgi, else default to eventlet
 # TODO(mtreinish): Remove the eventlet path here and in all the similar
 # conditionals below after the Pike release
@@ -96,29 +153,179 @@ function is_glance_enabled {
 # cleanup_glance() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
 function cleanup_glance {
-    # delete image files (glance)
-    sudo rm -rf $GLANCE_CACHE_DIR $GLANCE_IMAGE_DIR
+    # delete image files (glance) and all of the glance-remote temporary
+    # storage
+    sudo rm -rf $GLANCE_CACHE_DIR $GLANCE_IMAGE_DIR "${DATA_DIR}/glance-remote"
+
+    # Cleanup multiple stores directories
+    if [[ "$GLANCE_ENABLE_MULTIPLE_STORES" == "True" ]]; then
+        local store file_dir
+        for store in $(echo $GLANCE_MULTIPLE_FILE_STORES | tr "," "\n"); do
+            file_dir="${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/${store}/"
+            sudo rm -rf $file_dir
+        done
+
+        # Cleanup reserved stores directories
+        sudo rm -rf $GLANCE_STAGING_DIR $GLANCE_TASKS_DIR
+    fi
+}
+
+# Set multiple cinder store related config options for each of the cinder store
+#
+function configure_multiple_cinder_stores {
+
+    local be be_name be_type enabled_backends
+    for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
+        be_type=${be%%:*}
+        be_name=${be##*:}
+        enabled_backends+="${be_name}:cinder,"
+
+        set_common_cinder_store_params $be_name
+        iniset $GLANCE_API_CONF $be_name cinder_volume_type ${be_name}
+        if [[ "$be_type" == "nfs" ]]; then
+            mkdir -p "$GLANCE_NFS_MOUNTPOINT"
+            iniset $GLANCE_API_CONF $be_name cinder_mount_point_base "$GLANCE_NFS_MOUNTPOINT"
+        fi
+    done
+    iniset $GLANCE_API_CONF DEFAULT enabled_backends ${enabled_backends::-1}
+    iniset $GLANCE_API_CONF glance_store default_backend $GLANCE_CINDER_DEFAULT_BACKEND
+}
+
+# Set common cinder store options to given config section
+#
+# Arguments:
+# config_section
+#
+function set_common_cinder_store_params {
+    local config_section="$1"
+    iniset $GLANCE_API_CONF $config_section cinder_store_auth_address $KEYSTONE_SERVICE_URI_V3
+    iniset $GLANCE_API_CONF $config_section cinder_store_user_name glance
+    iniset $GLANCE_API_CONF $config_section cinder_store_password $SERVICE_PASSWORD
+    iniset $GLANCE_API_CONF $config_section cinder_store_project_name $SERVICE_PROJECT_NAME
+}
+
+# Configure multiple file stores options for each file store
+#
+# Arguments:
+#
+function configure_multiple_file_stores {
+    local store enabled_backends
+    enabled_backends=""
+    for store in $(echo $GLANCE_MULTIPLE_FILE_STORES | tr "," "\n"); do
+        enabled_backends+="${store}:file,"
+    done
+    iniset $GLANCE_API_CONF DEFAULT enabled_backends ${enabled_backends::-1}
+
+    # Glance multiple store Store specific configs
+    iniset $GLANCE_API_CONF glance_store default_backend $GLANCE_DEFAULT_BACKEND
+    local store
+    for store in $(echo $glance_multiple_file_stores | tr "," "\n"); do
+        iniset $GLANCE_API_CONF $store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/${store}/"
+    done
+}
+
+# Set reserved stores for glance
+function configure_reserved_stores {
+    iniset $GLANCE_API_CONF os_glance_staging_store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/os_glance_staging_store/"
+    iniset $GLANCE_API_CONF os_glance_tasks_store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/os_glance_tasks_store/"
+}
+
+# Copy rootwrap file from glance_store/etc/glance to /etc/glance
+#
+# Arguments:
+# source_path Source path to copy rootwrap files from
+#
+function copy_rootwrap {
+    local source_path="$1"
+    # Make glance configuration directory if it is not exists
+    sudo install -d -o $STACK_USER $GLANCE_CONF_DIR
+    cp -r $source_path/rootwrap.* $GLANCE_CONF_DIR/
+}
+
+# Set glance_store related config options
+#
+# Arguments:
+# USE_CINDER_FOR_GLANCE
+# GLANCE_ENABLE_MULTIPLE_STORES
+#
+function configure_glance_store {
+    local use_cinder_for_glance="$1"
+    local glance_enable_multiple_stores="$2"
+    local be
+
+    if [[ "$glance_enable_multiple_stores" == "False" ]]; then
+        # Configure traditional glance_store
+        if [[ "$use_cinder_for_glance" == "True" ]]; then
+            # set common glance_store parameters
+            iniset $GLANCE_API_CONF glance_store stores "cinder,file,http"
+            iniset $GLANCE_API_CONF glance_store default_store cinder
+
+            # set cinder related store parameters
+            set_common_cinder_store_params glance_store
+            # set nfs mount_point dir
+            for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
+                local be_name=${be##*:}
+                if [[ "$be_name" == "nfs" ]]; then
+                    mkdir -p $GLANCE_NFS_MOUNTPOINT
+                    iniset $GLANCE_API_CONF glance_store cinder_mount_point_base $GLANCE_NFS_MOUNTPOINT
+                fi
+            done
+        fi
+        # Store specific configs
+        iniset $GLANCE_API_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/
+    else
+        if [[ "$use_cinder_for_glance" == "True" ]]; then
+            # Configure multiple cinder stores for glance
+            configure_multiple_cinder_stores
+        else
+            # Configure multiple file stores for glance
+            configure_multiple_file_stores
+        fi
+        # Configure reserved stores
+        configure_reserved_stores
+    fi
+}
+
+function configure_glance_quotas {
+
+    # Registered limit resources in keystone are system-specific resources.
+    # Make sure we use a system-scoped token to interact with this API.
+
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL --region $REGION_NAME image_size_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit $GLANCE_LIMIT_IMAGE_SIZE_TOTAL --region $REGION_NAME image_stage_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit 100 --region $REGION_NAME image_count_total
+    openstack --os-cloud devstack-system-admin registered limit create --service glance \
+        --default-limit 100 --region $REGION_NAME image_count_uploading
+
+    # Tell glance to use these limits
+    iniset $GLANCE_API_CONF DEFAULT use_keystone_limits True
+
+    # Configure oslo_limit so it can talk to keystone
+    iniset $GLANCE_API_CONF oslo_limit user_domain_name $SERVICE_DOMAIN_NAME
+    iniset $GLANCE_API_CONF oslo_limit password $SERVICE_PASSWORD
+    iniset $GLANCE_API_CONF oslo_limit username glance
+    iniset $GLANCE_API_CONF oslo_limit auth_type password
+    iniset $GLANCE_API_CONF oslo_limit auth_url $KEYSTONE_SERVICE_URI
+    iniset $GLANCE_API_CONF oslo_limit system_scope all
+    iniset $GLANCE_API_CONF oslo_limit endpoint_id \
+           $(openstack --os-cloud devstack-system-admin endpoint list --service glance -f value -c ID)
+
+    # Allow the glance service user to read quotas
+    openstack --os-cloud devstack-system-admin role add --user glance \
+        --user-domain $SERVICE_DOMAIN_NAME --system all reader
 }

 # configure_glance() - Set config files, create data dirs, etc
 function configure_glance {
    sudo install -d -o $STACK_USER $GLANCE_CONF_DIR $GLANCE_METADEF_DIR

-    # Set non-default configuration options for registry
-    iniset $GLANCE_REGISTRY_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
-    iniset $GLANCE_REGISTRY_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
-    iniset $GLANCE_REGISTRY_CONF DEFAULT workers $API_WORKERS
+    # Set non-default configuration options for the API server
    local dburl
    dburl=`database_connection_url glance`
-    iniset $GLANCE_REGISTRY_CONF database connection $dburl
-    iniset $GLANCE_REGISTRY_CONF DEFAULT use_syslog $SYSLOG
-    iniset $GLANCE_REGISTRY_CONF paste_deploy flavor keystone
-    configure_keystone_authtoken_middleware $GLANCE_REGISTRY_CONF glance
-    iniset $GLANCE_REGISTRY_CONF oslo_messaging_notifications driver messagingv2
-    iniset_rpc_backend glance $GLANCE_REGISTRY_CONF
-    iniset $GLANCE_REGISTRY_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"

-    # Set non-default configuration options for the API server
    iniset $GLANCE_API_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $GLANCE_API_CONF database connection $dburl
    iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
@@ -128,23 +335,15 @@ function configure_glance {
    configure_keystone_authtoken_middleware $GLANCE_API_CONF glance
    iniset $GLANCE_API_CONF oslo_messaging_notifications driver messagingv2
    iniset_rpc_backend glance $GLANCE_API_CONF
-    if [ "$VIRT_DRIVER" = 'xenserver' ]; then
-        iniset $GLANCE_API_CONF DEFAULT container_formats "ami,ari,aki,bare,ovf,tgz"
-        iniset $GLANCE_API_CONF DEFAULT disk_formats "ami,ari,aki,vhd,raw,iso"
-    fi
    if [ "$VIRT_DRIVER" = 'libvirt' ] && [ "$LIBVIRT_TYPE" = 'parallels' ]; then
        iniset $GLANCE_API_CONF DEFAULT disk_formats "ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,ploop"
    fi
+    # Only use these if you know what you are doing!  See OSSN-0065
+    iniset $GLANCE_API_CONF DEFAULT show_image_direct_url $GLANCE_SHOW_DIRECT_URL
+    iniset $GLANCE_API_CONF DEFAULT show_multiple_locations $GLANCE_SHOW_MULTIPLE_LOCATIONS

-    # NOTE(flaper87): To uncomment as soon as all services consuming Glance are
-    # able to consume V2 entirely.
-    if [ "$GLANCE_V1_ENABLED" != "True" ]; then
-        iniset $GLANCE_API_CONF DEFAULT enable_v1_api False
-    fi
-
-    # Store specific configs
-    iniset $GLANCE_API_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/
-    iniset $GLANCE_API_CONF DEFAULT registry_host $(ipv6_unquote $GLANCE_SERVICE_HOST)
+    # Configure glance_store
+    configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES

    # CORS feature support - to allow calls from Horizon by default
    if [ -n "$GLANCE_CORS_ALLOWED_ORIGIN" ]; then
@@ -153,30 +352,27 @@ function configure_glance {
        iniset $GLANCE_API_CONF cors allowed_origin "http://$SERVICE_HOST"
    fi

-    # Store the images in swift if enabled.
-    if is_service_enabled s-proxy; then
-        iniset $GLANCE_API_CONF glance_store default_store swift
-        iniset $GLANCE_API_CONF glance_store swift_store_create_container_on_put True
-        if python3_enabled; then
-            iniset $GLANCE_API_CONF glance_store swift_store_auth_insecure True
-        fi
+    # No multiple stores for swift yet
+    if [[ "$GLANCE_ENABLE_MULTIPLE_STORES" == "False" ]]; then
+        # Store the images in swift if enabled.
+        if is_service_enabled s-proxy; then
+            iniset $GLANCE_API_CONF glance_store default_store swift
+            iniset $GLANCE_API_CONF glance_store swift_store_create_container_on_put True

-        iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF
-        iniset $GLANCE_API_CONF glance_store default_swift_reference ref1
-        iniset $GLANCE_API_CONF glance_store stores "file, http, swift"
-        iniset $GLANCE_API_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"
+            iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF
+            iniset $GLANCE_API_CONF glance_store default_swift_reference ref1
+            iniset $GLANCE_API_CONF glance_store stores "file, http, swift"
+            if is_service_enabled tls-proxy; then
+                iniset $GLANCE_API_CONF glance_store swift_store_cacert $SSL_BUNDLE_FILE
+            fi
+            iniset $GLANCE_API_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT"

-        iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_PROJECT_NAME:glance-swift
+            iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_PROJECT_NAME:glance-swift

-        iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD
-        if python3_enabled; then
-            # NOTE(dims): Currently the glance_store+swift does not support either an insecure flag
-            # or ability to specify the CACERT. So fallback to http:// url
-            iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_address ${KEYSTONE_SERVICE_URI/https/http}/v3
-        else
+            iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD
            iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_address $KEYSTONE_SERVICE_URI/v3
+            iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_version 3
        fi
-        iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_version 3
    fi

    # We need to tell glance what it's public endpoint is so that the version
@@ -185,32 +381,22 @@ function configure_glance {

    if is_service_enabled tls-proxy; then
        iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
-        iniset $GLANCE_REGISTRY_CONF DEFAULT bind_port $GLANCE_REGISTRY_PORT_INT
-
-        iniset $GLANCE_API_CONF keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
-        iniset $GLANCE_REGISTRY_CONF keystone_authtoken identity_uri $KEYSTONE_AUTH_URI
-    fi
-
-    if is_service_enabled tls-proxy; then
-        iniset $GLANCE_API_CONF DEFAULT registry_client_protocol https
+        iniset $GLANCE_API_CONF keystone_authtoken identity_uri $KEYSTONE_SERVICE_URI
    fi

    # Format logging
    setup_logging $GLANCE_API_CONF
-    setup_logging $GLANCE_REGISTRY_CONF

-    cp -p $GLANCE_DIR/etc/glance-registry-paste.ini $GLANCE_REGISTRY_PASTE_INI
    cp -p $GLANCE_DIR/etc/glance-api-paste.ini $GLANCE_API_PASTE_INI

    # Set non-default configuration options for the glance-cache
    iniset $GLANCE_CACHE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $GLANCE_CACHE_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
-    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_AUTH_URI
+    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_SERVICE_URI
    iniset $GLANCE_CACHE_CONF DEFAULT admin_tenant_name $SERVICE_PROJECT_NAME
    iniset $GLANCE_CACHE_CONF DEFAULT admin_user glance
    iniset $GLANCE_CACHE_CONF DEFAULT admin_password $SERVICE_PASSWORD
-    iniset $GLANCE_CACHE_CONF DEFAULT registry_host $(ipv6_unquote $GLANCE_SERVICE_HOST)

    # Store specific confs
    iniset $GLANCE_CACHE_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/
@@ -220,7 +406,6 @@ function configure_glance {
    iniset $GLANCE_IMAGE_IMPORT_CONF inject_metadata_properties ignore_user_roles admin
    iniset $GLANCE_IMAGE_IMPORT_CONF inject_metadata_properties inject

-    cp -p $GLANCE_DIR/etc/policy.json $GLANCE_POLICY_JSON
    cp -p $GLANCE_DIR/etc/schema-image.json $GLANCE_SCHEMA_JSON

    cp -p $GLANCE_DIR/etc/metadefs/*.json $GLANCE_METADEF_DIR
@@ -233,12 +418,25 @@ function configure_glance {
        iniset $GLANCE_CACHE_CONF DEFAULT cinder_endpoint_template "https://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/%(project_id)s"
    fi

-    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
+    if [[ "$GLANCE_STANDALONE" == False ]]; then
        write_local_uwsgi_http_config "$GLANCE_UWSGI_CONF" "$GLANCE_UWSGI" "/image"
+        # Grab our uwsgi listen address and use that to fill out our
+        # worker_self_reference_url config
+        iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url \
+               $(awk '-F= ' '/^http-socket/ { print "http://"$2}' \
+                    $GLANCE_UWSGI_CONF)
    else
+        write_local_proxy_http_config glance "http://$GLANCE_SERVICE_HOST:$GLANCE_SERVICE_PORT_INT" "/image"
        iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
+        iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
        iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
    fi
+
+    if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $GLANCE_API_CONF oslo_policy enforce_scope true
+        iniset $GLANCE_API_CONF oslo_policy enforce_new_defaults true
+        iniset $GLANCE_API_CONF DEFAULT enforce_secure_rbac true
+    fi
 }

 # create_glance_accounts() - Set up common required glance accounts
@@ -269,6 +467,11 @@ function create_glance_accounts {
        service_domain_id=$(get_or_create_domain $SERVICE_DOMAIN_NAME)
        iniset $GLANCE_SWIFT_STORE_CONF ref1 project_domain_id $service_domain_id
        iniset $GLANCE_SWIFT_STORE_CONF ref1 user_domain_id $service_domain_id
+
+        if [[ "$GLANCE_ENABLE_QUOTAS" = True ]]; then
+            configure_glance_quotas
+        fi
+
    fi
 }

@@ -301,11 +504,26 @@ function install_glanceclient {

 # install_glance() - Collect source and prepare
 function install_glance {
+    local glance_store_extras=()
+
+    if is_service_enabled cinder; then
+        glance_store_extras=("cinder" "${glance_store_extras[@]}")
+    fi
+
+    if is_service_enabled swift; then
+        glance_store_extras=("swift" "${glance_store_extras[@]}")
+    fi
+
    # Install glance_store from git so we make sure we're testing
    # the latest code.
    if use_library_from_git "glance_store"; then
        git_clone_by_name "glance_store"
-        setup_dev_lib "glance_store"
+        setup_dev_lib "glance_store" $(join_extras "${glance_store_extras[@]}")
+        copy_rootwrap ${DEST}/glance_store/etc/glance
+    else
+        # we still need to pass extras
+        pip_install_gr_extras glance-store $(join_extras "${glance_store_extras[@]}")
+        copy_rootwrap $GLANCE_STORE_ROOTWRAP_BASE_DIR
    fi

    git_clone $GLANCE_REPO $GLANCE_DIR $GLANCE_BRANCH
@@ -313,6 +531,72 @@ function install_glance {
    setup_develop $GLANCE_DIR
 }

+# glance_remote_conf() - Return the path to an alternate config file for
+#                        the remote glance clone
+function glance_remote_conf {
+    echo $(dirname "${GLANCE_CONF_DIR}")/glance-remote/$(basename "$1")
+}
+
+# start_glance_remote_clone() - Clone the regular glance api worker
+function start_glance_remote_clone {
+    local glance_remote_conf_dir glance_remote_port remote_data
+    local glance_remote_uwsgi
+
+    glance_remote_conf_dir="$(glance_remote_conf "")"
+    glance_remote_port=$(get_random_port)
+    glance_remote_uwsgi="$(glance_remote_conf $GLANCE_UWSGI_CONF)"
+
+    # Clone the existing ready-to-go glance-api setup
+    sudo rm -Rf "$glance_remote_conf_dir"
+    sudo cp -r "$GLANCE_CONF_DIR" "$glance_remote_conf_dir"
+    sudo chown $STACK_USER -R "$glance_remote_conf_dir"
+
+    # Point this worker at different data dirs
+    remote_data="${DATA_DIR}/glance-remote"
+    mkdir -p $remote_data/os_glance_tasks_store \
+          "${remote_data}/os_glance_staging_store"
+    iniset $(glance_remote_conf "$GLANCE_API_CONF") os_glance_staging_store \
+           filesystem_store_datadir "${remote_data}/os_glance_staging_store"
+    iniset $(glance_remote_conf "$GLANCE_API_CONF") os_glance_tasks_store \
+           filesystem_store_datadir "${remote_data}/os_glance_tasks_store"
+
+    # Point this worker to use different cache dir
+    mkdir -p "$remote_data/cache"
+    iniset $(glance_remote_conf "$GLANCE_API_CONF") DEFAULT \
+           image_cache_dir "${remote_data}/cache"
+
+    # Change our uwsgi to our new port
+    sed -ri "s/^(http-socket.*):[0-9]+/\1:$glance_remote_port/" \
+        "$glance_remote_uwsgi"
+
+    # Update the self-reference url with our new port
+    iniset $(glance_remote_conf $GLANCE_API_CONF) DEFAULT \
+           worker_self_reference_url \
+           $(awk '-F= ' '/^http-socket/ { print "http://"$2 }' \
+                    "$glance_remote_uwsgi")
+
+    # We need to create the systemd service for the clone, but then
+    # change it to include an Environment line to point the WSGI app
+    # at the alternate config directory.
+    write_uwsgi_user_unit_file devstack@g-api-r.service "$(which uwsgi) \
+                               --procname-prefix \
+                               glance-api-remote \
+                               --ini $glance_remote_uwsgi" \
+                               "" "$STACK_USER"
+    iniset -sudo ${SYSTEMD_DIR}/devstack@g-api-r.service \
+           "Service" "Environment" \
+           "OS_GLANCE_CONFIG_DIR=$glance_remote_conf_dir"
+
+    # Reload and restart with the new config
+    $SYSTEMCTL daemon-reload
+    $SYSTEMCTL restart devstack@g-api-r
+
+    get_or_create_service glance_remote image_remote "Alternate glance"
+    get_or_create_endpoint image_remote $REGION_NAME \
+                $(awk '-F= ' '/^http-socket/ { print "http://"$2 }' \
+                    $glance_remote_uwsgi)
+}
+
 # start_glance() - Start running processes
 function start_glance {
    local service_protocol=$GLANCE_SERVICE_PROTOCOL
@@ -320,16 +604,19 @@ function start_glance {
        if [[ "$WSGI_MODE" != "uwsgi" ]]; then
            start_tls_proxy glance-service '*' $GLANCE_SERVICE_PORT $GLANCE_SERVICE_HOST $GLANCE_SERVICE_PORT_INT
        fi
-        start_tls_proxy glance-registry '*' $GLANCE_REGISTRY_PORT $GLANCE_SERVICE_HOST $GLANCE_REGISTRY_PORT_INT
    fi

-    run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf"
-    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
-        run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
+    if [[ "$GLANCE_STANDALONE" == False ]]; then
+        run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF"
    else
        run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR"
    fi

+    if is_service_enabled g-api-r; then
+        echo "Starting the g-api-r clone service..."
+        start_glance_remote_clone
+    fi
+
    echo "Waiting for g-api ($GLANCE_SERVICE_HOST) to start..."
    if ! wait_for_service $SERVICE_TIMEOUT $GLANCE_URL; then
        die $LINENO "g-api did not start"
@@ -339,7 +626,7 @@ function start_glance {
 # stop_glance() - Stop running processes
 function stop_glance {
    stop_process g-api
-    stop_process g-reg
+    stop_process g-api-r
 }

 # Restore xtrace
@@ -29,7 +29,7 @@ GITDIR["pbr"]=$DEST/pbr
 # install_infra() - Collect source and prepare
 function install_infra {
    local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv"
-    [ ! -d $PIP_VIRTUAL_ENV ] && virtualenv $PIP_VIRTUAL_ENV
+    [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV
    # We don't care about testing git pbr in the requirements venv.
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR
@@ -9,7 +9,6 @@
 # - ``tls`` file
 # - ``DEST``, ``STACK_USER``
 # - ``FILES``
-# - ``IDENTITY_API_VERSION``
 # - ``BASE_SQL_CONN``
 # - ``SERVICE_HOST``, ``SERVICE_PROTOCOL``
 # - ``S3_SERVICE_PORT`` (template backend only)
@@ -50,9 +49,7 @@ fi
 KEYSTONE_CONF_DIR=${KEYSTONE_CONF_DIR:-/etc/keystone}
 KEYSTONE_CONF=$KEYSTONE_CONF_DIR/keystone.conf
 KEYSTONE_PUBLIC_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-public.ini
-KEYSTONE_ADMIN_UWSGI_CONF=$KEYSTONE_CONF_DIR/keystone-uwsgi-admin.ini
 KEYSTONE_PUBLIC_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-public
-KEYSTONE_ADMIN_UWSGI=$KEYSTONE_BIN_DIR/keystone-wsgi-admin

 # KEYSTONE_DEPLOY defines how keystone is deployed, allowed values:
 # - mod_wsgi : Run keystone under Apache HTTPd mod_wsgi
@@ -81,21 +78,12 @@ KEYSTONE_RESOURCE_BACKEND=${KEYSTONE_RESOURCE_BACKEND:-sql}
 KEYSTONE_TOKEN_FORMAT=${KEYSTONE_TOKEN_FORMAT:-fernet}
 KEYSTONE_TOKEN_FORMAT=$(echo ${KEYSTONE_TOKEN_FORMAT} | tr '[:upper:]' '[:lower:]')

-# Set Keystone interface configuration
-KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
-KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
-KEYSTONE_AUTH_PORT_INT=${KEYSTONE_AUTH_PORT_INT:-35358}
-KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
-
 # Public facing bits
 KEYSTONE_SERVICE_HOST=${KEYSTONE_SERVICE_HOST:-$SERVICE_HOST}
 KEYSTONE_SERVICE_PORT=${KEYSTONE_SERVICE_PORT:-5000}
 KEYSTONE_SERVICE_PORT_INT=${KEYSTONE_SERVICE_PORT_INT:-5001}
 KEYSTONE_SERVICE_PROTOCOL=${KEYSTONE_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}

-# Bind hosts
-KEYSTONE_ADMIN_BIND_HOST=${KEYSTONE_ADMIN_BIND_HOST:-$KEYSTONE_SERVICE_HOST}
-
 # Set the project for service accounts in Keystone
 SERVICE_DOMAIN_NAME=${SERVICE_DOMAIN_NAME:-Default}
 SERVICE_PROJECT_NAME=${SERVICE_PROJECT_NAME:-service}
@@ -106,7 +94,6 @@ SERVICE_TENANT_NAME=${SERVICE_PROJECT_NAME:-service}

 # if we are running with SSL use https protocols
 if is_service_enabled tls-proxy; then
-    KEYSTONE_AUTH_PROTOCOL="https"
    KEYSTONE_SERVICE_PROTOCOL="https"
 fi

@@ -115,7 +102,7 @@ KEYSTONE_SERVICE_URI=${KEYSTONE_SERVICE_PROTOCOL}://${KEYSTONE_SERVICE_HOST}/ide
 KEYSTONE_AUTH_URI=$KEYSTONE_SERVICE_URI

 # V3 URIs
-KEYSTONE_AUTH_URI_V3=$KEYSTONE_AUTH_URI/v3
+KEYSTONE_AUTH_URI_V3=$KEYSTONE_SERVICE_URI/v3
 KEYSTONE_SERVICE_URI_V3=$KEYSTONE_SERVICE_URI/v3

 # Security compliance
@@ -134,6 +121,15 @@ KEYSTONE_PASSWORD_HASH_ROUNDS=${KEYSTONE_PASSWORD_HASH_ROUNDS:-4}
 # Cache settings
 KEYSTONE_ENABLE_CACHE=${KEYSTONE_ENABLE_CACHE:-True}

+# Whether to create a keystone admin endpoint for legacy applications
+KEYSTONE_ADMIN_ENDPOINT=$(trueorfalse False KEYSTONE_ADMIN_ENDPOINT)
+
+# Flag to set the oslo_policy.enforce_scope. This is used to switch
+# the Identity API policies to start checking the scope of token. By Default,
+# this flag is False.
+# For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
+KEYSTONE_ENFORCE_SCOPE=$(trueorfalse False KEYSTONE_ENFORCE_SCOPE)
+
 # Functions
 # ---------

@@ -154,11 +150,8 @@ function cleanup_keystone {
        sudo rm -f $(apache_site_config_for keystone)
    else
        stop_process "keystone"
-        # TODO: remove admin at pike-2
        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI"
-        remove_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI"
        sudo rm -f $(apache_site_config_for keystone-wsgi-public)
-        sudo rm -f $(apache_site_config_for keystone-wsgi-admin)
    fi
 }

@@ -171,12 +164,10 @@ function _config_keystone_apache_wsgi {
    local keystone_certfile=""
    local keystone_keyfile=""
    local keystone_service_port=$KEYSTONE_SERVICE_PORT
-    local keystone_auth_port=$KEYSTONE_AUTH_PORT
    local venv_path=""

    if is_service_enabled tls-proxy; then
        keystone_service_port=$KEYSTONE_SERVICE_PORT_INT
-        keystone_auth_port=$KEYSTONE_AUTH_PORT_INT
    fi
    if [[ ${USE_VENV} = True ]]; then
        venv_path="python-path=${PROJECT_VENV["keystone"]}/lib/$(python_version)/site-packages"
@@ -185,7 +176,6 @@ function _config_keystone_apache_wsgi {
    sudo cp $FILES/apache-keystone.template $keystone_apache_conf
    sudo sed -e "
        s|%PUBLICPORT%|$keystone_service_port|g;
-        s|%ADMINPORT%|$keystone_auth_port|g;
        s|%APACHE_NAME%|$APACHE_NAME|g;
        s|%SSLLISTEN%|$keystone_ssl_listen|g;
        s|%SSLENGINE%|$keystone_ssl|g;
@@ -223,22 +213,17 @@ function configure_keystone {
    iniset_rpc_backend keystone $KEYSTONE_CONF oslo_messaging_notifications

    local service_port=$KEYSTONE_SERVICE_PORT
-    local auth_port=$KEYSTONE_AUTH_PORT

    if is_service_enabled tls-proxy; then
        # Set the service ports for a proxy to take the originals
        service_port=$KEYSTONE_SERVICE_PORT_INT
-        auth_port=$KEYSTONE_AUTH_PORT_INT
    fi

-    # Override the endpoints advertised by keystone (the public_endpoint and
-    # admin_endpoint) so that clients use the correct endpoint. By default, the
-    # keystone server uses the public_port and admin_port which isn't going to
-    # work when you want to use a different port (in the case of proxy), or you
-    # don't want the port (in the case of putting keystone on a path in
-    # apache).
+    # Override the endpoints advertised by keystone so that clients use the correct
+    # endpoint. By default, the keystone server uses the public_port which isn't
+    # going to work when you want to use a different port (in the case of proxy),
+    # or you don't want the port (in the case of putting keystone on a path in apache).
    iniset $KEYSTONE_CONF DEFAULT public_endpoint $KEYSTONE_SERVICE_URI
-    iniset $KEYSTONE_CONF DEFAULT admin_endpoint $KEYSTONE_AUTH_URI

    if [[ "$KEYSTONE_TOKEN_FORMAT" != "" ]]; then
        iniset $KEYSTONE_CONF token provider $KEYSTONE_TOKEN_FORMAT
@@ -261,7 +246,6 @@ function configure_keystone {
        _config_keystone_apache_wsgi
    else # uwsgi
        write_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI" "/identity"
-        write_uwsgi_config "$KEYSTONE_ADMIN_UWSGI_CONF" "$KEYSTONE_ADMIN_UWSGI" "/identity_admin"
    fi

    iniset $KEYSTONE_CONF DEFAULT max_token_size 16384
@@ -281,6 +265,11 @@ function configure_keystone {
        iniset $KEYSTONE_CONF security_compliance lockout_duration $KEYSTONE_LOCKOUT_DURATION
        iniset $KEYSTONE_CONF security_compliance unique_last_password_count $KEYSTONE_UNIQUE_LAST_PASSWORD_COUNT
    fi
+    if [[ "$KEYSTONE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
+        iniset $KEYSTONE_CONF oslo_policy enforce_scope true
+        iniset $KEYSTONE_CONF oslo_policy enforce_new_defaults true
+        iniset $KEYSTONE_CONF oslo_policy policy_file policy.yaml
+    fi
 }

 # create_keystone_accounts() - Sets up common required keystone accounts
@@ -303,40 +292,48 @@ function configure_keystone {
 # admins               admin            admin                 admin
 # nonadmins            demo, alt_demo   member, anotherrole   demo, alt_demo

+# System               User            Roles
+# ------------------------------------------------------------------
+# all                  admin           admin
+# all                  system_reader   reader
+# all                  system_member   member
+

 # Migrated from keystone_data.sh
 function create_keystone_accounts {

    # The keystone bootstrapping process (performed via keystone-manage
-    # bootstrap) creates an admin user, admin role, member role, and admin
+    # bootstrap) creates an admin user and an admin
    # project. As a sanity check we exercise the CLI to retrieve the IDs for
    # these values.
    local admin_project
    admin_project=$(openstack project show "admin" -f value -c id)
    local admin_user
    admin_user=$(openstack user show "admin" -f value -c id)
+    # These roles are also created during bootstrap but we don't need their IDs
    local admin_role="admin"
    local member_role="member"
+    local reader_role="reader"

-    get_or_add_user_domain_role $admin_role $admin_user default
+    async_run ks-domain-role get_or_add_user_domain_role $admin_role $admin_user default

    # Create service project/role
    get_or_create_domain "$SERVICE_DOMAIN_NAME"
-    get_or_create_project "$SERVICE_PROJECT_NAME" "$SERVICE_DOMAIN_NAME"
+    async_run ks-project get_or_create_project "$SERVICE_PROJECT_NAME" "$SERVICE_DOMAIN_NAME"

    # Service role, so service users do not have to be admins
-    get_or_create_role service
+    async_run ks-service get_or_create_role service

    # The ResellerAdmin role is used by Nova and Ceilometer so we need to keep it.
    # The admin role in swift allows a user to act as an admin for their project,
    # but ResellerAdmin is needed for a user to act as any project. The name of this
    # role is also configurable in swift-proxy.conf
-    get_or_create_role ResellerAdmin
+    async_run ks-reseller get_or_create_role ResellerAdmin

    # another_role demonstrates that an arbitrary role may be created and used
    # TODO(sleepsonthefloor): show how this can be used for rbac in the future!
    local another_role="anotherrole"
-    get_or_create_role $another_role
+    async_run ks-anotherrole get_or_create_role $another_role

    # invisible project - admin can't see this one
    local invis_project
@@ -349,21 +346,55 @@ function create_keystone_accounts {
    demo_user=$(get_or_create_user "demo" \
        "$ADMIN_PASSWORD" "default" "demo@example.com")

-    get_or_add_user_project_role $member_role $demo_user $demo_project
-    get_or_add_user_project_role $admin_role $admin_user $demo_project
-    get_or_add_user_project_role $another_role $demo_user $demo_project
-    get_or_add_user_project_role $member_role $demo_user $invis_project
+    async_wait ks-{domain-role,domain,project,service,reseller,anotherrole}

-    # alt_demo
+    async_run ks-demo-member get_or_add_user_project_role $member_role $demo_user $demo_project
+
+    async_run ks-demo-admin get_or_add_user_project_role $admin_role $admin_user $demo_project
+    async_run ks-demo-another get_or_add_user_project_role $another_role $demo_user $demo_project
+    async_run ks-demo-invis get_or_add_user_project_role $member_role $demo_user $invis_project
+
+    # Create a user to act as a reader on project demo
+    local demo_reader
+    demo_reader=$(get_or_create_user "demo_reader" \
+        "$ADMIN_PASSWORD" "default" "demo_reader@example.com")
+
+    async_run ks-demo-reader get_or_add_user_project_role $reader_role $demo_reader $demo_project
+
+    # Create a different project called alt_demo
    local alt_demo_project
    alt_demo_project=$(get_or_create_project "alt_demo" default)
+    # Create a user to act as member, admin and anotherrole on project alt_demo
    local alt_demo_user
    alt_demo_user=$(get_or_create_user "alt_demo" \
        "$ADMIN_PASSWORD" "default" "alt_demo@example.com")

-    get_or_add_user_project_role $member_role $alt_demo_user $alt_demo_project
-    get_or_add_user_project_role $admin_role $admin_user $alt_demo_project
-    get_or_add_user_project_role $another_role $alt_demo_user $alt_demo_project
+    async_run ks-alt-admin get_or_add_user_project_role $admin_role $alt_demo_user $alt_demo_project
+    async_run ks-alt-another get_or_add_user_project_role $another_role $alt_demo_user $alt_demo_project
+
+    # Create another user to act as a member on project alt_demo
+    local alt_demo_member
+    alt_demo_member=$(get_or_create_user "alt_demo_member" \
+        "$ADMIN_PASSWORD" "default" "alt_demo_member@example.com")
+    async_run ks-alt-member-user get_or_add_user_project_role $member_role $alt_demo_member $alt_demo_project
+
+    # Create another user to act as a reader on project alt_demo
+    local alt_demo_reader
+    alt_demo_reader=$(get_or_create_user "alt_demo_reader" \
+        "$ADMIN_PASSWORD" "default" "alt_demo_reader@example.com")
+    async_run ks-alt-reader-user get_or_add_user_project_role $reader_role $alt_demo_reader $alt_demo_project
+
+    # Create two users, give one the member role on the system and the other the
+    # reader role on the system. These two users model system-member and
+    # system-reader personas. The admin user already has the admin role on the
+    # system and we can re-use this user as a system-admin.
+    system_member_user=$(get_or_create_user "system_member" \
+        "$ADMIN_PASSWORD" "default" "system_member@example.com")
+    async_run ks-system-member get_or_add_user_system_role $member_role $system_member_user "all"
+
+    system_reader_user=$(get_or_create_user "system_reader" \
+        "$ADMIN_PASSWORD" "default" "system_reader@example.com")
+    async_run ks-system-reader get_or_add_user_system_role $reader_role $system_reader_user "all"

    # groups
    local admin_group
@@ -373,11 +404,16 @@ function create_keystone_accounts {
    non_admin_group=$(get_or_create_group "nonadmins" \
        "default" "non-admin group")

-    get_or_add_group_project_role $member_role $non_admin_group $demo_project
-    get_or_add_group_project_role $another_role $non_admin_group $demo_project
-    get_or_add_group_project_role $member_role $non_admin_group $alt_demo_project
-    get_or_add_group_project_role $another_role $non_admin_group $alt_demo_project
-    get_or_add_group_project_role $admin_role $admin_group $admin_project
+    async_run ks-group-memberdemo get_or_add_group_project_role $member_role $non_admin_group $demo_project
+    async_run ks-group-anotherdemo get_or_add_group_project_role $another_role $non_admin_group $demo_project
+    async_run ks-group-memberalt get_or_add_group_project_role $member_role $non_admin_group $alt_demo_project
+    async_run ks-group-anotheralt get_or_add_group_project_role $another_role $non_admin_group $alt_demo_project
+    async_run ks-group-admin get_or_add_group_project_role $admin_role $admin_group $admin_project
+
+    async_wait ks-demo-{member,admin,another,invis,reader}
+    async_wait ks-alt-{admin,another,member-user,reader-user}
+    async_wait ks-system-{member,reader}
+    async_wait ks-group-{memberdemo,anotherdemo,memberalt,anotheralt,admin}

    if is_service_enabled ldap; then
        create_ldap_domain
@@ -413,6 +449,7 @@ function configure_keystone_authtoken_middleware {
    local section=${3:-keystone_authtoken}

    iniset $conf_file $section auth_type password
+    iniset $conf_file $section interface public
    iniset $conf_file $section auth_url $KEYSTONE_SERVICE_URI
    iniset $conf_file $section username $admin_user
    iniset $conf_file $section password $SERVICE_PASSWORD
@@ -421,7 +458,7 @@ function configure_keystone_authtoken_middleware {
    iniset $conf_file $section project_domain_name "$SERVICE_DOMAIN_NAME"

    iniset $conf_file $section cafile $SSL_BUNDLE_FILE
-    iniset $conf_file $section memcached_servers localhost:11211
+    iniset $conf_file $section memcached_servers $MEMCACHE_SERVERS
 }

 # configure_auth_token_middleware conf_file admin_user IGNORED [section]
@@ -504,8 +541,6 @@ function install_keystone {

    if [ "$KEYSTONE_DEPLOY" == "mod_wsgi" ]; then
        install_apache_wsgi
-    elif [ "$KEYSTONE_DEPLOY" == "uwsgi" ]; then
-        pip_install uwsgi
    fi
 }

@@ -513,7 +548,7 @@ function install_keystone {
 function start_keystone {
    # Get right service port for testing
    local service_port=$KEYSTONE_SERVICE_PORT
-    local auth_protocol=$KEYSTONE_AUTH_PROTOCOL
+    local auth_protocol=$KEYSTONE_SERVICE_PROTOCOL
    if is_service_enabled tls-proxy; then
        service_port=$KEYSTONE_SERVICE_PORT_INT
        auth_protocol="http"
@@ -523,7 +558,7 @@ function start_keystone {
        enable_apache_site keystone
        restart_apache_server
    else # uwsgi
-        run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" ""
+        run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" ""
    fi

    echo "Waiting for keystone to start..."
@@ -532,7 +567,7 @@ function start_keystone {
    # unencryted traffic at this point.
    # If running in Apache, use the path rather than port.

-    local service_uri=$auth_protocol://$KEYSTONE_SERVICE_HOST/identity/v$IDENTITY_API_VERSION/
+    local service_uri=$auth_protocol://$KEYSTONE_SERVICE_HOST/identity/v3/

    if ! wait_for_service $SERVICE_TIMEOUT $service_uri; then
        die $LINENO "keystone did not start"
@@ -541,7 +576,6 @@ function start_keystone {
    # Start proxies if enabled
    if is_service_enabled tls-proxy; then
        start_tls_proxy keystone-service '*' $KEYSTONE_SERVICE_PORT $KEYSTONE_SERVICE_HOST $KEYSTONE_SERVICE_PORT_INT
-        start_tls_proxy keystone-auth '*' $KEYSTONE_AUTH_PORT $KEYSTONE_AUTH_HOST $KEYSTONE_AUTH_PORT_INT
    fi

    # (re)start memcached to make sure we have a clean memcache.
@@ -562,12 +596,8 @@ function stop_keystone {
 # This function uses the following GLOBAL variables:
 # - ``KEYSTONE_BIN_DIR``
 # - ``ADMIN_PASSWORD``
-# - ``IDENTITY_API_VERSION``
-# - ``KEYSTONE_AUTH_URI``
 # - ``REGION_NAME``
-# - ``KEYSTONE_SERVICE_PROTOCOL``
-# - ``KEYSTONE_SERVICE_HOST``
-# - ``KEYSTONE_SERVICE_PORT``
+# - ``KEYSTONE_SERVICE_URI``
 function bootstrap_keystone {
    $KEYSTONE_BIN_DIR/keystone-manage bootstrap \
        --bootstrap-username admin \
@@ -576,8 +606,16 @@ function bootstrap_keystone {
        --bootstrap-role-name admin \
        --bootstrap-service-name keystone \
        --bootstrap-region-id "$REGION_NAME" \
-        --bootstrap-admin-url "$KEYSTONE_AUTH_URI" \
        --bootstrap-public-url "$KEYSTONE_SERVICE_URI"
+    if [ "$KEYSTONE_ADMIN_ENDPOINT" == "True" ]; then
+        openstack endpoint create --region "$REGION_NAME" \
+            --os-username admin \
+            --os-user-domain-id default \
+            --os-password "$ADMIN_PASSWORD" \
+            --os-project-name admin \
+            --os-project-domain-id default \
+            keystone admin "$KEYSTONE_SERVICE_URI"
+    fi
 }

 # create_ldap_domain() - Create domain file and initialize domain with a user
@@ -33,14 +33,17 @@ LDAP_SERVICE_NAME=slapd

 if is_ubuntu; then
    LDAP_OLCDB_NUMBER=1
+    LDAP_OLCDB_TYPE=mdb
    LDAP_ROOTPW_COMMAND=replace
 elif is_fedora; then
    LDAP_OLCDB_NUMBER=2
+    LDAP_OLCDB_TYPE=hdb
    LDAP_ROOTPW_COMMAND=add
 elif is_suse; then
    # SUSE has slappasswd in /usr/sbin/
    PATH=$PATH:/usr/sbin/
    LDAP_OLCDB_NUMBER=1
+    LDAP_OLCDB_TYPE=hdb
    LDAP_ROOTPW_COMMAND=add
    LDAP_SERVICE_NAME=ldap
 fi
@@ -56,6 +59,7 @@ function _ldap_varsubst {
    local slappass=$2
    sed -e "
        s|\${LDAP_OLCDB_NUMBER}|$LDAP_OLCDB_NUMBER|
+        s|\${LDAP_OLCDB_TYPE}|$LDAP_OLCDB_TYPE|
        s|\${SLAPPASS}|$slappass|
        s|\${LDAP_ROOTPW_COMMAND}|$LDAP_ROOTPW_COMMAND|
        s|\${BASE_DC}|$LDAP_BASE_DC|
@@ -157,7 +161,7 @@ function configure_ldap {
    slapd slapd/dump_database_destdir string /var/backups/slapd-VERSION
    slapd slapd/domain string Users
    slapd shared/organization string $LDAP_DOMAIN
-    slapd slapd/backend string HDB
+    slapd slapd/backend string ${LDAP_OLCDB_TYPE^^}
    slapd slapd/purge_database boolean true
    slapd slapd/move_old_database boolean true
    slapd slapd/allow_ldap_v2 boolean false
@@ -38,6 +38,7 @@ GITDIR["oslo.config"]=$DEST/oslo.config
 GITDIR["oslo.context"]=$DEST/oslo.context
 GITDIR["oslo.db"]=$DEST/oslo.db
 GITDIR["oslo.i18n"]=$DEST/oslo.i18n
+GITDIR["oslo.limit"]=$DEST/oslo.limit
 GITDIR["oslo.log"]=$DEST/oslo.log
 GITDIR["oslo.messaging"]=$DEST/oslo.messaging
 GITDIR["oslo.middleware"]=$DEST/oslo.middleware
@@ -59,6 +60,7 @@ GITDIR["tooz"]=$DEST/tooz
 # Non oslo libraries are welcomed below as well, this prevents
 # duplication of this code.
 GITDIR["os-brick"]=$DEST/os-brick
+GITDIR["os-resource-classes"]=$DEST/os-resource-classes
 GITDIR["os-traits"]=$DEST/os-traits

 # Support entry points installation of console scripts
@@ -72,7 +74,7 @@ function _install_lib_from_source {
    local name=$1
    if use_library_from_git "$name"; then
        git_clone_by_name "$name"
-        setup_dev_lib "$name"
+        setup_dev_lib -bindep "$name"
    fi
 }

@@ -101,6 +103,7 @@ function install_libs {
    _install_lib_from_source "oslo.context"
    _install_lib_from_source "oslo.db"
    _install_lib_from_source "oslo.i18n"
+    _install_lib_from_source "oslo.limit"
    _install_lib_from_source "oslo.log"
    _install_lib_from_source "oslo.messaging"
    _install_lib_from_source "oslo.middleware"
@@ -122,6 +125,7 @@ function install_libs {
    #
    # os-traits for nova
    _install_lib_from_source "os-brick"
+    _install_lib_from_source "os-resource-classes"
    _install_lib_from_source "os-traits"
    #
    # python client libraries we might need from git can go here
@@ -53,28 +53,10 @@ function _remove_lvm_volume_group {
    sudo vgremove -f $vg
 }

-# _clean_lvm_backing_file() removes the backing file of the
-# volume group
-#
-# Usage: _clean_lvm_backing_file() $backing_file
-function _clean_lvm_backing_file {
-    local backing_file=$1
-
-    # If the backing physical device is a loop device, it was probably setup by DevStack
-    if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
-        local vg_dev
-        vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}')
-        if [[ -n "$vg_dev" ]]; then
-            sudo losetup -d $vg_dev
-        fi
-        rm -f $backing_file
-    fi
-}
-
 # clean_lvm_volume_group() cleans up the volume group and removes the
 # backing file
 #
-# Usage: clean_lvm_volume_group $vg
+# Usage: clean_lvm_volume_group() $vg
 function clean_lvm_volume_group {
    local vg=$1

@@ -83,11 +65,22 @@ function clean_lvm_volume_group {
    # if there is no logical volume left, it's safe to attempt a cleanup
    # of the backing file
    if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
-        _clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX
+        local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX
+
+        if [[ -n "$vg$BACKING_FILE_SUFFIX" ]] && \
+           [[ -e "/etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service" ]]; then
+            sudo systemctl disable --now $vg$BACKING_FILE_SUFFIX.service
+            sudo rm -f /etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service
+            sudo systemctl daemon-reload
+        fi
+
+        # If the backing physical device is a loop device, it was probably setup by DevStack
+        if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
+            rm -f $backing_file
+        fi
    fi
 }

-
 # _create_lvm_volume_group creates default volume group
 #
 # Usage: _create_lvm_volume_group() $vg $size
@@ -106,8 +99,20 @@ function _create_lvm_volume_group {
            directio="--direct-io=on"
        fi

+        # Only create systemd service if it doesn't already exists
+        if [[ ! -e "/etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service" ]]; then
+            sed -e "
+                s|%DIRECTIO%|${directio}|g;
+                s|%BACKING_FILE%|${backing_file}|g;
+            " $FILES/lvm-backing-file.template | sudo tee \
+                /etc/systemd/system/$vg$BACKING_FILE_SUFFIX.service
+
+            sudo systemctl daemon-reload
+            sudo systemctl enable --now $vg$BACKING_FILE_SUFFIX.service
+        fi
+
        local vg_dev
-        vg_dev=$(sudo losetup -f --show $directio $backing_file)
+        vg_dev=$(sudo losetup --associated $backing_file -O NAME -n)

        # Only create volume group if it doesn't already exist
        if ! sudo vgs $vg; then
@@ -124,13 +129,9 @@ function init_lvm_volume_group {
    local vg=$1
    local size=$2

-    # Start the lvmetad and tgtd services
-    if is_fedora || is_suse; then
-        # services is not started by default
-        start_service lvm2-lvmetad
-        if [ "$CINDER_ISCSI_HELPER" = "tgtadm" ]; then
-            start_service tgtd
-        fi
+    # Start the tgtd service on Fedora and SUSE if tgtadm is used
+    if  is_fedora || is_suse  && [[ "$CINDER_ISCSI_HELPER" = "tgtadm" ]]; then
+        start_service tgtd
    fi

    # Start with a clean volume group
@@ -37,6 +37,11 @@ NEUTRON_DEPLOY_MOD_WSGI=$(trueorfalse False NEUTRON_DEPLOY_MOD_WSGI)
 NEUTRON_AGENT=${NEUTRON_AGENT:-openvswitch}
 NEUTRON_DIR=$DEST/neutron

+# If NEUTRON_ENFORCE_SCOPE == True, it will set "enforce_scope"
+# and "enforce_new_defaults" to True in the Neutron's config to enforce usage
+# of the new RBAC policies and scopes.
+NEUTRON_ENFORCE_SCOPE=$(trueorfalse False NEUTRON_ENFORCE_SCOPE)
+
 NEUTRON_DISTRIBUTED_ROUTING=$(trueorfalse False NEUTRON_DISTRIBUTED_ROUTING)
 # Distributed Virtual Router (DVR) configuration
 # Can be:
@@ -109,6 +114,12 @@ NEUTRON_TENANT_VLAN_RANGE=${NEUTRON_TENANT_VLAN_RANGE:-${TENANT_VLAN_RANGE:-100:
 # Physical network for VLAN network usage.
 NEUTRON_PHYSICAL_NETWORK=${NEUTRON_PHYSICAL_NETWORK:-}

+# The name of the service in the endpoint URL
+NEUTRON_ENDPOINT_SERVICE_NAME=${NEUTRON_ENDPOINT_SERVICE_NAME-"networking"}
+if [[ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" && -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]]; then
+    NEUTRON_ENDPOINT_SERVICE_NAME="networking"
+fi
+

 # Additional neutron api config files
 declare -a -g _NEUTRON_SERVER_EXTRA_CONF_FILES_ABS
@@ -141,6 +152,7 @@ fi
 # cleanup_neutron() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
 function cleanup_neutron_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    source $TOP_DIR/lib/neutron_plugins/${NEUTRON_AGENT}_agent
    if is_neutron_ovs_base_plugin; then
        neutron_ovs_base_cleanup
@@ -164,6 +176,7 @@ function configure_root_helper_options {

 # configure_neutron() - Set config files, create data dirs, etc
 function configure_neutron_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    sudo install -d -o $STACK_USER $NEUTRON_CONF_DIR

    (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)
@@ -206,7 +219,6 @@ function configure_neutron_new {
        iniset $NEUTRON_CONF DEFAULT core_plugin $NEUTRON_CORE_PLUGIN

        iniset $NEUTRON_CONF DEFAULT policy_file $policy_file
-        iniset $NEUTRON_CONF DEFAULT allow_overlapping_ips True
        iniset $NEUTRON_CONF DEFAULT router_distributed $NEUTRON_DISTRIBUTED_ROUTING

        iniset $NEUTRON_CONF DEFAULT auth_strategy $NEUTRON_AUTH_STRATEGY
@@ -222,7 +234,12 @@ function configure_neutron_new {
        else
            mech_drivers+=",linuxbridge"
        fi
+        if [[ "$mech_drivers" == *"linuxbridge"* ]]; then
+            iniset $NEUTRON_CONF experimental linuxbridge True
+        fi
+
        iniset $NEUTRON_CORE_PLUGIN_CONF ml2 mechanism_drivers $mech_drivers
+        iniset $NEUTRON_CORE_PLUGIN_CONF ml2 overlay_ip_version $TUNNEL_IP_VERSION

        iniset $NEUTRON_CORE_PLUGIN_CONF ml2_type_vxlan vni_ranges 1001:2000
        iniset $NEUTRON_CORE_PLUGIN_CONF ml2_type_flat flat_networks $PUBLIC_NETWORK_NAME
@@ -232,6 +249,7 @@ function configure_neutron_new {
        if [[ "$NEUTRON_PORT_SECURITY" = "True" ]]; then
            neutron_ml2_extension_driver_add port_security
        fi
+        configure_rbac_policies
    fi

    # Neutron OVS or LB agent
@@ -243,10 +261,10 @@ function configure_neutron_new {
        # Configure the neutron agent
        if [[ $NEUTRON_AGENT == "linuxbridge" ]]; then
            iniset $NEUTRON_CORE_PLUGIN_CONF securitygroup firewall_driver iptables
-            iniset $NEUTRON_CORE_PLUGIN_CONF vxlan local_ip $HOST_IP
+            iniset $NEUTRON_CORE_PLUGIN_CONF vxlan local_ip $TUNNEL_ENDPOINT_IP
        elif [[ $NEUTRON_AGENT == "openvswitch" ]]; then
            iniset $NEUTRON_CORE_PLUGIN_CONF securitygroup firewall_driver openvswitch
-            iniset $NEUTRON_CORE_PLUGIN_CONF ovs local_ip $HOST_IP
+            iniset $NEUTRON_CORE_PLUGIN_CONF ovs local_ip $TUNNEL_ENDPOINT_IP

            if [[ "$NEUTRON_DISTRIBUTED_ROUTING" = "True" ]]; then
                iniset $NEUTRON_CORE_PLUGIN_CONF agent l2_population True
@@ -353,8 +371,8 @@ function configure_neutron_rootwrap {
 # Takes a single optional argument which is the config file to update,
 # if not passed $NOVA_CONF is used.
 function configure_neutron_nova_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    local conf=${1:-$NOVA_CONF}
-    iniset $conf DEFAULT use_neutron True
    iniset $conf neutron auth_type "password"
    iniset $conf neutron auth_url "$KEYSTONE_SERVICE_URI"
    iniset $conf neutron username neutron
@@ -365,8 +383,6 @@ function configure_neutron_nova_new {
    iniset $conf neutron auth_strategy $NEUTRON_AUTH_STRATEGY
    iniset $conf neutron region_name "$REGION_NAME"

-    iniset $conf DEFAULT firewall_driver nova.virt.firewall.NoopFirewallDriver
-
    # optionally set options in nova_conf
    neutron_plugin_create_nova_conf $conf

@@ -382,13 +398,17 @@ function configure_neutron_nova_new {

 # create_neutron_accounts() - Create required service accounts
 function create_neutron_accounts_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    local neutron_url

    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
-        neutron_url=$NEUTRON_SERVICE_PROTOCOL://$NEUTRON_SERVICE_HOST/networking/
+        neutron_url=$NEUTRON_SERVICE_PROTOCOL://$NEUTRON_SERVICE_HOST/
    else
        neutron_url=$NEUTRON_SERVICE_PROTOCOL://$NEUTRON_SERVICE_HOST:$NEUTRON_SERVICE_PORT/
    fi
+    if [ ! -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]; then
+        neutron_url=$neutron_url$NEUTRON_ENDPOINT_SERVICE_NAME
+    fi


    if [[ "$ENABLED_SERVICES" =~ "neutron-api" ]]; then
@@ -405,6 +425,7 @@ function create_neutron_accounts_new {
 # init_neutron() - Initialize databases, etc.
 function init_neutron_new {

+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    recreate_database neutron

    time_start "dbsync"
@@ -415,6 +436,7 @@ function init_neutron_new {

 # install_neutron() - Collect source and prepare
 function install_neutron_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    git_clone $NEUTRON_REPO $NEUTRON_DIR $NEUTRON_BRANCH
    setup_develop $NEUTRON_DIR

@@ -466,20 +488,23 @@ function start_neutron_api {
    done

    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
-        run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
-        neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST/networking/
+        run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
+        neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST/
        enable_service neutron-rpc-server
        run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $opts"
    else
        # Start the Neutron service
        # TODO(sc68cal) Stop hard coding this
        run_process neutron-api "$NEUTRON_BIN_DIR/neutron-server $opts"
-        neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST:$service_port
+        neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST:$service_port/
        # Start proxy if enabled
        if is_service_enabled tls-proxy; then
            start_tls_proxy neutron '*' $NEUTRON_SERVICE_PORT $NEUTRON_SERVICE_HOST $NEUTRON_SERVICE_PORT_INT
        fi
    fi
+    if [ ! -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]; then
+        neutron_url=$neutron_url$NEUTRON_ENDPOINT_SERVICE_NAME
+    fi

    if ! wait_for_service $SERVICE_TIMEOUT $neutron_url; then
        die $LINENO "neutron-api did not start"
@@ -488,6 +513,7 @@ function start_neutron_api {

 # start_neutron() - Start running processes
 function start_neutron_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    # Start up the neutron agents if enabled
    # TODO(sc68cal) Make this pluggable so different DevStack plugins for different Neutron plugins
    # can resolve the $NEUTRON_AGENT_BINARY
@@ -525,6 +551,7 @@ function start_neutron_new {

 # stop_neutron() - Stop running processes
 function stop_neutron_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    for serv in neutron-api neutron-agent neutron-l3; do
        stop_process $serv
    done
@@ -540,13 +567,13 @@ function stop_neutron_new {
    fi

    if is_service_enabled neutron-metadata-agent; then
-        sudo pkill -9 -f neutron-ns-metadata-proxy || :
        stop_process neutron-metadata-agent
    fi
 }

 # neutron_service_plugin_class_add() - add service plugin class
 function neutron_service_plugin_class_add_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    local service_plugin_class=$1
    local plugins=""

@@ -571,11 +598,13 @@ function _neutron_ml2_extension_driver_add {
 }

 function neutron_server_config_add_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    _NEUTRON_SERVER_EXTRA_CONF_FILES_ABS+=($1)
 }

 # neutron_deploy_rootwrap_filters() - deploy rootwrap filters
 function neutron_deploy_rootwrap_filters_new {
+    deprecated "Using lib/neutron is deprecated, and it will be removed in AA release!"
    local srcdir=$1
    sudo install -d -o root -g root -m 755 $NEUTRON_CONF_DIR/rootwrap.d
    sudo install -o root -g root -m 644 $srcdir/etc/neutron/rootwrap.d/*.filters $NEUTRON_CONF_DIR/rootwrap.d
@@ -615,6 +644,19 @@ function configure_neutron {
    fi
 }

+# configure_rbac_policies() - Configure Neutron to enforce new RBAC
+# policies and scopes if NEUTRON_ENFORCE_SCOPE == True
+function configure_rbac_policies {
+    if [[ "$NEUTRON_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == "True" ]]; then
+        iniset $NEUTRON_CONF oslo_policy enforce_new_defaults True
+        iniset $NEUTRON_CONF oslo_policy enforce_scope True
+    else
+        iniset $NEUTRON_CONF oslo_policy enforce_new_defaults False
+        iniset $NEUTRON_CONF oslo_policy enforce_scope False
+    fi
+}
+
+
 function configure_neutron_nova {
    if is_neutron_legacy_enabled; then
        # Call back to old function
@@ -58,8 +58,6 @@
 # Neutron Network Configuration
 # -----------------------------

-deprecated "Using lib/neutron-legacy is deprecated, and it will be removed in the future"
-
 if is_service_enabled tls-proxy; then
    Q_PROTOCOL="https"
 fi
@@ -92,6 +90,11 @@ NEUTRON_DEPLOY_MOD_WSGI=$(trueorfalse False NEUTRON_DEPLOY_MOD_WSGI)

 NEUTRON_UWSGI_CONF=$NEUTRON_CONF_DIR/neutron-api-uwsgi.ini

+# If NEUTRON_ENFORCE_SCOPE == True, it will set "enforce_scope"
+# and "enforce_new_defaults" to True in the Neutron's config to enforce usage
+# of the new RBAC policies and scopes.
+NEUTRON_ENFORCE_SCOPE=$(trueorfalse False NEUTRON_ENFORCE_SCOPE)
+
 # Agent binaries.  Note, binary paths for other agents are set in per-service
 # scripts in lib/neutron_plugins/services/
 AGENT_DHCP_BINARY="$NEUTRON_BIN_DIR/neutron-dhcp-agent"
@@ -135,10 +138,19 @@ Q_NOTIFY_NOVA_PORT_DATA_CHANGES=${Q_NOTIFY_NOVA_PORT_DATA_CHANGES:-True}
 VIF_PLUGGING_IS_FATAL=${VIF_PLUGGING_IS_FATAL:-True}
 VIF_PLUGGING_TIMEOUT=${VIF_PLUGGING_TIMEOUT:-300}

+# Allow to skip stopping of OVN services
+SKIP_STOP_OVN=${SKIP_STOP_OVN:-False}
+
 # The directory which contains files for Q_PLUGIN_EXTRA_CONF_FILES.
 # /etc/neutron is assumed by many of devstack plugins.  Do not change.
 _Q_PLUGIN_EXTRA_CONF_PATH=/etc/neutron

+# The name of the service in the endpoint URL
+NEUTRON_ENDPOINT_SERVICE_NAME=${NEUTRON_ENDPOINT_SERVICE_NAME-"networking"}
+if [[ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" && -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]]; then
+    NEUTRON_ENDPOINT_SERVICE_NAME="networking"
+fi
+
 # List of config file names in addition to the main plugin config file
 # To add additional plugin config files, use ``neutron_server_config_add``
 # utility function.  For example:
@@ -228,22 +240,17 @@ PHYSICAL_NETWORK=${PHYSICAL_NETWORK:-public}
 # Example: ``OVS_PHYSICAL_BRIDGE=br-eth1``
 OVS_PHYSICAL_BRIDGE=${OVS_PHYSICAL_BRIDGE:-br-ex}

-default_route_dev=$(ip route | grep ^default | awk '{print $5}')
-die_if_not_set $LINENO default_route_dev "Failure retrieving default route device"
 # With the linuxbridge agent, if using VLANs for tenant networks,
 # or if using flat or VLAN provider networks, set in ``localrc`` to
 # the name of the network interface to use for the physical
 # network.
 #
 # Example: ``LB_PHYSICAL_INTERFACE=eth1``
-LB_PHYSICAL_INTERFACE=${LB_PHYSICAL_INTERFACE:-$default_route_dev}
-
-# When Neutron tunnels are enabled it is needed to specify the
-# IP address of the end point in the local server. This IP is set
-# by default to the same IP address that the HOST IP.
-# This variable can be used to specify a different end point IP address
-# Example: ``TUNNEL_ENDPOINT_IP=1.1.1.1``
-TUNNEL_ENDPOINT_IP=${TUNNEL_ENDPOINT_IP:-$HOST_IP}
+if [[ $Q_AGENT == "linuxbridge" && -z ${LB_PHYSICAL_INTERFACE} ]]; then
+    default_route_dev=$( (ip route; ip -6 route) | grep ^default | head -n 1 | awk '{print $5}')
+    die_if_not_set $LINENO default_route_dev "Failure retrieving default route device"
+    LB_PHYSICAL_INTERFACE=$default_route_dev
+fi

 # With the openvswitch plugin, set to True in ``localrc`` to enable
 # provider GRE tunnels when ``ENABLE_TENANT_TUNNELS`` is False.
@@ -275,6 +282,12 @@ source $TOP_DIR/lib/neutron_plugins/services/metering

 # L3 Service functions
 source $TOP_DIR/lib/neutron_plugins/services/l3
+
+# Additional Neutron service plugins
+source $TOP_DIR/lib/neutron_plugins/services/placement
+source $TOP_DIR/lib/neutron_plugins/services/trunk
+source $TOP_DIR/lib/neutron_plugins/services/qos
+
 # Use security group or not
 if has_neutron_plugin_security_group; then
    Q_USE_SECGROUP=${Q_USE_SECGROUP:-True}
@@ -364,6 +377,26 @@ function configure_mutnauq {
        _configure_neutron_ceilometer_notifications
    fi

+    if [[ $Q_AGENT == "ovn" ]]; then
+        configure_ovn
+        configure_ovn_plugin
+    fi
+
+    # Configure Neutron's advanced services
+    if is_service_enabled q-placement neutron-placement; then
+        configure_placement_extension
+    fi
+    if is_service_enabled q-trunk neutron-trunk; then
+        configure_trunk_extension
+    fi
+    if is_service_enabled q-qos neutron-qos; then
+        configure_qos
+        if is_service_enabled q-l3 neutron-l3; then
+            configure_l3_agent_extension_fip_qos
+            configure_l3_agent_extension_gateway_ip_qos
+        fi
+    fi
+
    iniset $NEUTRON_CONF DEFAULT api_workers "$API_WORKERS"
    # devstack is not a tool for running uber scale OpenStack
    # clouds, therefore running without a dedicated RPC worker
@@ -373,9 +406,8 @@ function configure_mutnauq {

 function create_nova_conf_neutron {
    local conf=${1:-$NOVA_CONF}
-    iniset $conf DEFAULT use_neutron True
    iniset $conf neutron auth_type "password"
-    iniset $conf neutron auth_url "$KEYSTONE_AUTH_URI"
+    iniset $conf neutron auth_url "$KEYSTONE_SERVICE_URI"
    iniset $conf neutron username "$Q_ADMIN_USERNAME"
    iniset $conf neutron password "$SERVICE_PASSWORD"
    iniset $conf neutron user_domain_name "$SERVICE_DOMAIN_NAME"
@@ -384,11 +416,6 @@ function create_nova_conf_neutron {
    iniset $conf neutron auth_strategy "$Q_AUTH_STRATEGY"
    iniset $conf neutron region_name "$REGION_NAME"

-    if [[ "$Q_USE_SECGROUP" == "True" ]]; then
-        LIBVIRT_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver
-        iniset $conf DEFAULT firewall_driver $LIBVIRT_FIREWALL_DRIVER
-    fi
-
    # optionally set options in nova_conf
    neutron_plugin_create_nova_conf $conf

@@ -410,10 +437,13 @@ function create_nova_conf_neutron {
 function create_mutnauq_accounts {
    local neutron_url
    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
-        neutron_url=$Q_PROTOCOL://$SERVICE_HOST/networking/
+        neutron_url=$Q_PROTOCOL://$SERVICE_HOST/
    else
        neutron_url=$Q_PROTOCOL://$SERVICE_HOST:$Q_PORT/
    fi
+    if [ ! -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]; then
+        neutron_url=$neutron_url$NEUTRON_ENDPOINT_SERVICE_NAME
+    fi

    if [[ "$ENABLED_SERVICES" =~ "q-svc" ]]; then

@@ -446,6 +476,10 @@ function install_mutnauq {

    git_clone $NEUTRON_REPO $NEUTRON_DIR $NEUTRON_BRANCH
    setup_develop $NEUTRON_DIR
+
+    if [[ $Q_AGENT == "ovn" ]]; then
+        install_ovn
+    fi
 }

 # install_neutron_agent_packages() - Collect source and prepare
@@ -465,6 +499,35 @@ function configure_neutron_after_post_config {
    if [[ $Q_SERVICE_PLUGIN_CLASSES != '' ]]; then
        iniset $NEUTRON_CONF DEFAULT service_plugins $Q_SERVICE_PLUGIN_CLASSES
    fi
+    configure_rbac_policies
+}
+
+# configure_rbac_policies() - Configure Neutron to enforce new RBAC
+# policies and scopes if NEUTRON_ENFORCE_SCOPE == True
+function configure_rbac_policies {
+    if [[ "$NEUTRON_ENFORCE_SCOPE" == "True" || "$ENFORCE_SCOPE" == True ]]; then
+        iniset $NEUTRON_CONF oslo_policy enforce_new_defaults True
+        iniset $NEUTRON_CONF oslo_policy enforce_scope True
+    else
+        iniset $NEUTRON_CONF oslo_policy enforce_new_defaults False
+        iniset $NEUTRON_CONF oslo_policy enforce_scope False
+    fi
+}
+
+# Start running OVN processes
+function start_ovn_services {
+    if [[ $Q_AGENT == "ovn" ]]; then
+        init_ovn
+        start_ovn
+        if [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]]; then
+            if [[ "$NEUTRON_CREATE_INITIAL_NETWORKS" != "True" ]]; then
+                echo "OVN_L3_CREATE_PUBLIC_NETWORK=True is being ignored "
+                echo "because NEUTRON_CREATE_INITIAL_NETWORKS is set to False"
+            else
+                create_public_bridge
+            fi
+        fi
+    fi
 }

 # Start running processes
@@ -483,18 +546,21 @@ function start_neutron_service_and_check {
    # Start the Neutron service
    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
        enable_service neutron-api
-        run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
-        neutron_url=$Q_PROTOCOL://$Q_HOST/networking/
+        run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
+        neutron_url=$Q_PROTOCOL://$Q_HOST/
        enable_service neutron-rpc-server
        run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options"
    else
        run_process q-svc "$NEUTRON_BIN_DIR/neutron-server $cfg_file_options"
-        neutron_url=$service_protocol://$Q_HOST:$service_port
+        neutron_url=$service_protocol://$Q_HOST:$service_port/
        # Start proxy if enabled
        if is_service_enabled tls-proxy; then
            start_tls_proxy neutron '*' $Q_PORT $Q_HOST $Q_PORT_INT
        fi
    fi
+    if [ ! -z "$NEUTRON_ENDPOINT_SERVICE_NAME" ]; then
+        neutron_url=$neutron_url$NEUTRON_ENDPOINT_SERVICE_NAME
+    fi
    echo "Waiting for Neutron to start..."

    local testcmd="wget ${ssl_ca} --no-proxy -q -O- $neutron_url"
@@ -524,11 +590,7 @@ function start_mutnauq_l2_agent {
 function start_mutnauq_other_agents {
    run_process q-dhcp "$AGENT_DHCP_BINARY --config-file $NEUTRON_CONF --config-file $Q_DHCP_CONF_FILE"

-    if is_service_enabled neutron-vpnaas; then
-        :  # Started by plugin
-    else
-        run_process q-l3 "$AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"
-    fi
+    run_process q-l3 "$AGENT_L3_BINARY $(determine_config_files neutron-l3-agent)"

    run_process q-meta "$AGENT_META_BINARY --config-file $NEUTRON_CONF --config-file $Q_META_CONF_FILE"
    run_process q-metering "$AGENT_METERING_BINARY --config-file $NEUTRON_CONF --config-file $METERING_AGENT_CONF_FILENAME"
@@ -566,7 +628,6 @@ function stop_mutnauq_other {
    fi

    if is_service_enabled q-meta; then
-        sudo pkill -9 -f neutron-ns-metadata-proxy || :
        stop_process q-meta
    fi

@@ -583,6 +644,10 @@ function stop_mutnauq_other {
 function stop_mutnauq {
    stop_mutnauq_other
    stop_mutnauq_l2_agent
+
+    if [[ $Q_AGENT == "ovn" && $SKIP_STOP_OVN != "True" ]]; then
+        stop_ovn
+    fi
 }

 # _move_neutron_addresses_route() - Move the primary IP to the OVS bridge
@@ -630,7 +695,7 @@ function _move_neutron_addresses_route {
            IP_UP="sudo ip link set $to_intf up"
            if [[ "$af" == "inet" ]]; then
                IP=$(echo $IP_BRD | awk '{ print $1; exit }' | grep -o -E '(.*)/' | cut -d "/" -f1)
-                ARP_CMD="sudo arping -A -c 3 -w 4.5 -I $to_intf $IP "
+                ARP_CMD="sudo arping -A -c 3 -w 5 -I $to_intf $IP "
            fi
        fi

@@ -640,6 +705,27 @@ function _move_neutron_addresses_route {
    fi
 }

+# _configure_public_network_connectivity() - Configures connectivity to the
+# external network using $PUBLIC_INTERFACE or NAT on the single interface
+# machines
+function _configure_public_network_connectivity {
+    # If we've given a PUBLIC_INTERFACE to take over, then we assume
+    # that we can own the whole thing, and privot it into the OVS
+    # bridge. If we are not, we're probably on a single interface
+    # machine, and we just setup NAT so that fixed guests can get out.
+    if [[ -n "$PUBLIC_INTERFACE" ]]; then
+        _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True False "inet"
+
+        if [[ $(ip -f inet6 a s dev "$PUBLIC_INTERFACE" | grep -c 'global') != 0 ]]; then
+            _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False False "inet6"
+        fi
+    else
+        for d in $default_v4_route_devs; do
+            sudo iptables -t nat -A POSTROUTING -o $d -s $FLOATING_RANGE -j MASQUERADE
+        done
+    fi
+}
+
 # cleanup_mutnauq() - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
 function cleanup_mutnauq {
@@ -676,6 +762,10 @@ function cleanup_mutnauq {
    for ns in $(sudo ip netns list | grep -o -E '(qdhcp|qrouter|fip|snat)-[0-9a-f-]*'); do
        sudo ip netns delete ${ns}
    done
+
+    if [[ $Q_AGENT == "ovn" ]]; then
+        cleanup_ovn
+    fi
 }


@@ -826,18 +916,30 @@ function _configure_neutron_plugin_agent {
    neutron_plugin_configure_plugin_agent
 }

+function _replace_api_paste_composite {
+    local sep
+    sep=$(echo -ne "\x01")
+    # Replace it
+    $sudo sed -i -e "s/\/\: neutronversions_composite/\/"${NEUTRON_ENDPOINT_SERVICE_NAME}"\/\: neutronversions_composite/" "$Q_API_PASTE_FILE"
+    $sudo sed -i -e "s/\/healthcheck\: healthcheck/\/"${NEUTRON_ENDPOINT_SERVICE_NAME}"\/healthcheck\: healthcheck/" "$Q_API_PASTE_FILE"
+    $sudo sed -i -e "s/\/v2.0\: neutronapi_v2_0/\/"${NEUTRON_ENDPOINT_SERVICE_NAME}"\/v2.0\: neutronapi_v2_0/" "$Q_API_PASTE_FILE"
+}
+
 # _configure_neutron_service() - Set config files for neutron service
 # It is called when q-svc is enabled.
 function _configure_neutron_service {
    Q_API_PASTE_FILE=$NEUTRON_CONF_DIR/api-paste.ini
    cp $NEUTRON_DIR/etc/api-paste.ini $Q_API_PASTE_FILE

+    if [[ -n "$NEUTRON_ENDPOINT_SERVICE_NAME" ]]; then
+        _replace_api_paste_composite
+    fi
+
    # Update either configuration file with plugin
    iniset $NEUTRON_CONF DEFAULT core_plugin $Q_PLUGIN_CLASS

    iniset $NEUTRON_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $NEUTRON_CONF oslo_policy policy_file $Q_POLICY_FILE
-    iniset $NEUTRON_CONF DEFAULT allow_overlapping_ips $Q_ALLOW_OVERLAPPING_IP

    iniset $NEUTRON_CONF DEFAULT auth_strategy $Q_AUTH_STRATEGY
    configure_keystone_authtoken_middleware $NEUTRON_CONF $Q_ADMIN_USERNAME
@@ -848,6 +950,9 @@ function _configure_neutron_service {

    configure_keystone_authtoken_middleware $NEUTRON_CONF nova nova

+    # Configuration for placement client
+    configure_keystone_authtoken_middleware $NEUTRON_CONF placement placement
+
    # Configure plugin
    neutron_plugin_configure_service
 }
@@ -974,6 +1079,15 @@ function _ssh_check_neutron {
    test_with_retry "$testcmd" "server $ip didn't become ssh-able" $timeout_sec
 }

+function plugin_agent_add_l2_agent_extension {
+    local l2_agent_extension=$1
+    if [[ -z "$L2_AGENT_EXTENSIONS" ]]; then
+        L2_AGENT_EXTENSIONS=$l2_agent_extension
+    elif [[ ! ,${L2_AGENT_EXTENSIONS}, =~ ,${l2_agent_extension}, ]]; then
+        L2_AGENT_EXTENSIONS+=",$l2_agent_extension"
+    fi
+}
+
 # Restore xtrace
 $_XTRACE_NEUTRON

@@ -38,7 +38,7 @@ function neutron_plugin_create_nova_conf {
 }

 function neutron_plugin_install_agent_packages {
-    install_package bridge-utils
+    :
 }

 function neutron_plugin_configure_dhcp_agent {
@@ -48,7 +48,7 @@ function neutron_plugin_configure_dhcp_agent {

 function neutron_plugin_configure_l3_agent {
    local conf_file=$1
-    sudo brctl addbr $PUBLIC_BRIDGE
+    sudo ip link add $PUBLIC_BRIDGE type bridge
    set_mtu $PUBLIC_BRIDGE $PUBLIC_BRIDGE_MTU
 }

@@ -7,9 +7,16 @@
 _XTRACE_NEUTRON_ML2=$(set +o | grep xtrace)
 set +o xtrace

+# Default OVN L2 agent
+Q_AGENT=${Q_AGENT:-ovn}
+if [ -f $TOP_DIR/lib/neutron_plugins/${Q_AGENT}_agent ]; then
+    source $TOP_DIR/lib/neutron_plugins/${Q_AGENT}_agent
+fi
+
 # Enable this to simply and quickly enable tunneling with ML2.
-# Select either 'gre', 'vxlan', or 'gre,vxlan'
-Q_ML2_TENANT_NETWORK_TYPE=${Q_ML2_TENANT_NETWORK_TYPE:-"vxlan"}
+# For ML2/OVS select either 'gre', 'vxlan', or 'gre,vxlan'.
+# For ML2/OVN use 'geneve'.
+Q_ML2_TENANT_NETWORK_TYPE=${Q_ML2_TENANT_NETWORK_TYPE:-"geneve"}
 # This has to be set here since the agent will set this in the config file
 if [[ "$Q_ML2_TENANT_NETWORK_TYPE" == "gre" || "$Q_ML2_TENANT_NETWORK_TYPE" == "vxlan" ]]; then
    Q_TUNNEL_TYPES=$Q_ML2_TENANT_NETWORK_TYPE
@@ -17,14 +24,8 @@ elif [[ "$ENABLE_TENANT_TUNNELS" == "True" ]]; then
    Q_TUNNEL_TYPES=gre
 fi

-# Default openvswitch L2 agent
-Q_AGENT=${Q_AGENT:-openvswitch}
-if [ -f $TOP_DIR/lib/neutron_plugins/${Q_AGENT}_agent ]; then
-    source $TOP_DIR/lib/neutron_plugins/${Q_AGENT}_agent
-fi
-
 # List of MechanismDrivers to load
-Q_ML2_PLUGIN_MECHANISM_DRIVERS=${Q_ML2_PLUGIN_MECHANISM_DRIVERS:-openvswitch,linuxbridge}
+Q_ML2_PLUGIN_MECHANISM_DRIVERS=${Q_ML2_PLUGIN_MECHANISM_DRIVERS:-ovn}
 # Default GRE TypeDriver options
 Q_ML2_PLUGIN_GRE_TYPE_OPTIONS=${Q_ML2_PLUGIN_GRE_TYPE_OPTIONS:-tunnel_id_ranges=$TENANT_TUNNEL_RANGES}
 # Default VXLAN TypeDriver options
@@ -124,6 +125,10 @@ function neutron_plugin_configure_service {
    fi

    populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 mechanism_drivers=$Q_ML2_PLUGIN_MECHANISM_DRIVERS
+    if [[ "$Q_ML2_PLUGIN_MECHANISM_DRIVERS" == *"linuxbridge"* ]]; then
+        iniset $NEUTRON_CONF experimental linuxbridge True
+    fi
+    populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 overlay_ip_version=$TUNNEL_IP_VERSION

    if [[ -n "$Q_ML2_PLUGIN_TYPE_DRIVERS" ]]; then
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2 type_drivers=$Q_ML2_PLUGIN_TYPE_DRIVERS
@@ -155,5 +160,9 @@ function has_neutron_plugin_security_group {
    return 0
 }

+function configure_qos_ml2 {
+    neutron_ml2_extension_driver_add "qos"
+}
+
 # Restore xtrace
 $_XTRACE_NEUTRON_ML2
@@ -11,8 +11,6 @@ function neutron_plugin_create_nova_conf {
    local conf="$1"
    NOVA_OVS_BRIDGE=${NOVA_OVS_BRIDGE:-"br-int"}
    iniset $conf neutron ovs_bridge $NOVA_OVS_BRIDGE
-    LIBVIRT_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver
-    iniset $conf DEFAULT firewall_driver $LIBVIRT_FIREWALL_DRIVER
 }

 function neutron_plugin_install_agent_packages {
@@ -15,6 +15,10 @@ function neutron_plugin_create_nova_conf {

 function neutron_plugin_install_agent_packages {
    _neutron_ovs_base_install_agent_packages
+    if use_library_from_git "os-ken"; then
+        git_clone_by_name "os-ken"
+        setup_dev_lib "os-ken"
+    fi
 }

 function neutron_plugin_configure_dhcp_agent {
@@ -41,8 +45,10 @@ function neutron_plugin_configure_plugin_agent {
    # Setup physical network bridge mappings.  Override
    # ``OVS_VLAN_RANGES`` and ``OVS_BRIDGE_MAPPINGS`` in ``localrc`` for more
    # complex physical network configurations.
-    if [[ "$OVS_BRIDGE_MAPPINGS" == "" ]] && [[ "$PHYSICAL_NETWORK" != "" ]] && [[ "$OVS_PHYSICAL_BRIDGE" != "" ]]; then
-        OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE
+    if [[ "$PHYSICAL_NETWORK" != "" ]] && [[ "$OVS_PHYSICAL_BRIDGE" != "" ]]; then
+        if [[ "$OVS_BRIDGE_MAPPINGS" == "" ]]; then
+            OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE
+        fi

        # Configure bridge manually with physical interface as port for multi-node
        _neutron_ovs_base_add_bridge $OVS_PHYSICAL_BRIDGE
@@ -0,0 +1,821 @@
+#!/bin/bash
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+# Global Sources
+# --------------
+
+# There are some ovs functions OVN depends on that must be sourced from
+# the ovs neutron plugins.
+source ${TOP_DIR}/lib/neutron_plugins/ovs_base
+source ${TOP_DIR}/lib/neutron_plugins/openvswitch_agent
+
+# Load devstack ovs compliation and loading functions
+source ${TOP_DIR}/lib/neutron_plugins/ovs_source
+
+# Set variables for building OVN from source
+OVN_REPO=${OVN_REPO:-https://github.com/ovn-org/ovn.git}
+OVN_REPO_NAME=$(basename ${OVN_REPO} | cut -f1 -d'.')
+OVN_REPO_NAME=${OVN_REPO_NAME:-ovn}
+OVN_BRANCH=${OVN_BRANCH:-v20.06.1}
+# The commit removing OVN bits from the OVS tree, it is the commit that is not
+# present in OVN tree and is used to distinguish if OVN is part of OVS or not.
+# https://github.com/openvswitch/ovs/commit/05bf1dbb98b0635a51f75e268ef8aed27601401d
+OVN_SPLIT_HASH=05bf1dbb98b0635a51f75e268ef8aed27601401d
+
+if is_service_enabled tls-proxy; then
+    OVN_PROTO=ssl
+else
+    OVN_PROTO=tcp
+fi
+
+# How to connect to ovsdb-server hosting the OVN SB database.
+OVN_SB_REMOTE=${OVN_SB_REMOTE:-$OVN_PROTO:$SERVICE_HOST:6642}
+
+# How to connect to ovsdb-server hosting the OVN NB database
+OVN_NB_REMOTE=${OVN_NB_REMOTE:-$OVN_PROTO:$SERVICE_HOST:6641}
+
+# ml2/config for neutron_sync_mode
+OVN_NEUTRON_SYNC_MODE=${OVN_NEUTRON_SYNC_MODE:-log}
+
+# Configured DNS servers to be used with internal_dns extension, only
+# if the subnet DNS is not configured.
+OVN_DNS_SERVERS=${OVN_DNS_SERVERS:-8.8.8.8}
+
+# The type of OVN L3 Scheduler to use. The OVN L3 Scheduler determines the
+# hypervisor/chassis where a routers gateway should be hosted in OVN. The
+# default OVN L3 scheduler is leastloaded
+OVN_L3_SCHEDULER=${OVN_L3_SCHEDULER:-leastloaded}
+
+# A UUID to uniquely identify this system.  If one is not specified, a random
+# one will be generated.  A randomly generated UUID will be saved in a file
+# $OVS_SYSCONFDIR/system-id.conf (typically /etc/openvswitch/system-id.conf)
+# so that the same one will be re-used if you re-run DevStack or restart
+# Open vSwitch service.
+OVN_UUID=${OVN_UUID:-}
+
+# Whether or not to build the openvswitch kernel module from ovs.  This is required
+# unless the distro kernel includes ovs+conntrack support.
+OVN_BUILD_MODULES=$(trueorfalse False OVN_BUILD_MODULES)
+OVN_BUILD_FROM_SOURCE=$(trueorfalse False OVN_BUILD_FROM_SOURCE)
+if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+    Q_BUILD_OVS_FROM_GIT=True
+fi
+
+# Whether or not to install the ovs python module from ovs source.  This can be
+# used to test and validate new ovs python features.  This should only be used
+# for development purposes since the ovs python version is controlled by OpenStack
+# requirements.
+OVN_INSTALL_OVS_PYTHON_MODULE=$(trueorfalse False OVN_INSTALL_OVS_PYTHON_MODULE)
+
+# GENEVE overlay protocol overhead. Defaults to 38 bytes plus the IP version
+# overhead (20 bytes for IPv4 (default) or 40 bytes for IPv6) which is determined
+# based on the ML2 overlay_ip_version option. The ML2 framework will use this to
+# configure the MTU DHCP option.
+OVN_GENEVE_OVERHEAD=${OVN_GENEVE_OVERHEAD:-38}
+
+# The log level of the OVN databases (north and south).
+# Supported log levels are: off, emer, err, warn, info or dbg.
+# More information about log levels can be found at
+# http://www.openvswitch.org/support/dist-docs/ovs-appctl.8.txt
+OVN_DBS_LOG_LEVEL=${OVN_DBS_LOG_LEVEL:-info}
+
+OVN_META_CONF=$NEUTRON_CONF_DIR/neutron_ovn_metadata_agent.ini
+OVN_META_DATA_HOST=${OVN_META_DATA_HOST:-$(ipv6_unquote $SERVICE_HOST)}
+
+# If True (default) the node will be considered a gateway node.
+ENABLE_CHASSIS_AS_GW=$(trueorfalse True ENABLE_CHASSIS_AS_GW)
+OVN_L3_CREATE_PUBLIC_NETWORK=$(trueorfalse True OVN_L3_CREATE_PUBLIC_NETWORK)
+
+export OVSDB_SERVER_LOCAL_HOST=$SERVICE_LOCAL_HOST
+TUNNEL_IP=$TUNNEL_ENDPOINT_IP
+if [[ "$SERVICE_IP_VERSION" == 6 ]]; then
+    OVSDB_SERVER_LOCAL_HOST=[$OVSDB_SERVER_LOCAL_HOST]
+    TUNNEL_IP=[$TUNNEL_IP]
+fi
+
+OVN_IGMP_SNOOPING_ENABLE=$(trueorfalse False OVN_IGMP_SNOOPING_ENABLE)
+
+OVS_PREFIX=
+if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+    OVS_PREFIX=/usr/local
+fi
+OVS_SBINDIR=$OVS_PREFIX/sbin
+OVS_BINDIR=$OVS_PREFIX/bin
+OVS_RUNDIR=$OVS_PREFIX/var/run/openvswitch
+OVS_SHAREDIR=$OVS_PREFIX/share/openvswitch
+OVS_SCRIPTDIR=$OVS_SHAREDIR/scripts
+OVS_DATADIR=$DATA_DIR/ovs
+OVS_SYSCONFDIR=${OVS_SYSCONFDIR:-$OVS_PREFIX/etc/openvswitch}
+
+if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+    OVN_DATADIR=$DATA_DIR/ovn
+else
+    # When using OVN from packages, the data dir for OVN DBs is
+    # /var/lib/ovn
+    OVN_DATADIR=/var/lib/ovn
+fi
+OVN_SHAREDIR=$OVS_PREFIX/share/ovn
+OVN_SCRIPTDIR=$OVN_SHAREDIR/scripts
+OVN_RUNDIR=$OVS_PREFIX/var/run/ovn
+
+NEUTRON_OVN_BIN_DIR=$(get_python_exec_prefix)
+NEUTRON_OVN_METADATA_BINARY="neutron-ovn-metadata-agent"
+
+STACK_GROUP="$( id --group --name "$STACK_USER" )"
+
+OVN_NORTHD_SERVICE=ovn-northd.service
+if is_ubuntu; then
+    # The ovn-central.service file on Ubuntu is responsible for starting
+    # ovn-northd and the OVN DBs (on CentOS this is done by ovn-northd.service)
+    OVN_NORTHD_SERVICE=ovn-central.service
+fi
+OVSDB_SERVER_SERVICE=ovsdb-server.service
+OVS_VSWITCHD_SERVICE=ovs-vswitchd.service
+OVN_CONTROLLER_SERVICE=ovn-controller.service
+OVN_CONTROLLER_VTEP_SERVICE=ovn-controller-vtep.service
+if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+    OVSDB_SERVER_SERVICE=devstack@ovsdb-server.service
+    OVS_VSWITCHD_SERVICE=devstack@ovs-vswitchd.service
+    OVN_NORTHD_SERVICE=devstack@ovn-northd.service
+    OVN_CONTROLLER_SERVICE=devstack@ovn-controller.service
+    OVN_CONTROLLER_VTEP_SERVICE=devstack@ovn-controller-vtep.service
+fi
+
+# Defaults Overwrite
+# ------------------
+
+Q_ML2_PLUGIN_MECHANISM_DRIVERS=${Q_ML2_PLUGIN_MECHANISM_DRIVERS:-ovn,logger}
+Q_ML2_PLUGIN_TYPE_DRIVERS=${Q_ML2_PLUGIN_TYPE_DRIVERS:-local,flat,vlan,geneve}
+Q_ML2_TENANT_NETWORK_TYPE=${Q_ML2_TENANT_NETWORK_TYPE:-"geneve"}
+Q_ML2_PLUGIN_GENEVE_TYPE_OPTIONS=${Q_ML2_PLUGIN_GENEVE_TYPE_OPTIONS:-"vni_ranges=1:65536"}
+Q_ML2_PLUGIN_EXT_DRIVERS=${Q_ML2_PLUGIN_EXT_DRIVERS:-port_security,qos}
+# this one allows empty:
+ML2_L3_PLUGIN=${ML2_L3_PLUGIN-"ovn-router"}
+
+Q_LOG_DRIVER_RATE_LIMIT=${Q_LOG_DRIVER_RATE_LIMIT:-100}
+Q_LOG_DRIVER_BURST_LIMIT=${Q_LOG_DRIVER_BURST_LIMIT:-25}
+Q_LOG_DRIVER_LOG_BASE=${Q_LOG_DRIVER_LOG_BASE:-acl_log_meter}
+
+# Utility Functions
+# -----------------
+
+function wait_for_db_file {
+    local count=0
+    while [ ! -f $1 ]; do
+        sleep 1
+        count=$((count+1))
+        if [ "$count" -gt 40 ]; then
+            die $LINENO "DB File $1 not found"
+        fi
+    done
+}
+
+function wait_for_sock_file {
+    local count=0
+    while [ ! -S $1 ]; do
+        sleep 1
+        count=$((count+1))
+        if [ "$count" -gt 40 ]; then
+            die $LINENO "Socket $1 not found"
+        fi
+    done
+}
+
+function use_new_ovn_repository {
+    if [[ "$OVN_BUILD_FROM_SOURCE" == "False" ]]; then
+        return 0
+    fi
+    if [ -z "$is_new_ovn" ]; then
+        local ovs_repo_dir=$DEST/$OVS_REPO_NAME
+        if [ ! -d $ovs_repo_dir ]; then
+            git_timed clone $OVS_REPO $ovs_repo_dir
+            pushd $ovs_repo_dir
+            git checkout $OVS_BRANCH
+            popd
+        else
+            clone_repository $OVS_REPO $ovs_repo_dir $OVS_BRANCH
+        fi
+        # Check the split commit exists in the current branch
+        pushd $ovs_repo_dir
+        git log $OVS_BRANCH --pretty=format:"%H" | grep -q $OVN_SPLIT_HASH
+        is_new_ovn=$?
+        popd
+    fi
+    return $is_new_ovn
+}
+
+# NOTE(rtheis): Function copied from DevStack _neutron_ovs_base_setup_bridge
+# and _neutron_ovs_base_add_bridge with the call to neutron-ovs-cleanup
+# removed. The call is not relevant for OVN, as it is specific to the use
+# of Neutron's OVS agent and hangs when running stack.sh because
+# neutron-ovs-cleanup uses the OVSDB native interface.
+function ovn_base_setup_bridge {
+    local bridge=$1
+    local addbr_cmd="sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge -- set bridge $bridge protocols=OpenFlow13,OpenFlow15"
+
+    if [ "$OVS_DATAPATH_TYPE" != "system" ] ; then
+        addbr_cmd="$addbr_cmd -- set Bridge $bridge datapath_type=${OVS_DATAPATH_TYPE}"
+    fi
+
+    $addbr_cmd
+    sudo ovs-vsctl --no-wait br-set-external-id $bridge bridge-id $bridge
+}
+
+function _start_process {
+    $SYSTEMCTL daemon-reload
+    $SYSTEMCTL enable $1
+    $SYSTEMCTL restart $1
+}
+
+function _run_process {
+    local service=$1
+    local cmd="$2"
+    local stop_cmd="$3"
+    local group=$4
+    local user=${5:-$STACK_USER}
+
+    local systemd_service="devstack@$service.service"
+    local unit_file="$SYSTEMD_DIR/$systemd_service"
+    local environment="OVN_RUNDIR=$OVS_RUNDIR OVN_DBDIR=$OVN_DATADIR OVN_LOGDIR=$LOGDIR OVS_RUNDIR=$OVS_RUNDIR OVS_DBDIR=$OVS_DATADIR OVS_LOGDIR=$LOGDIR"
+
+    echo "Starting $service executed command": $cmd
+
+    write_user_unit_file $systemd_service "$cmd" "$group" "$user"
+    iniset -sudo $unit_file "Service" "Type" "forking"
+    iniset -sudo $unit_file "Service" "RemainAfterExit" "yes"
+    iniset -sudo $unit_file "Service" "KillMode" "mixed"
+    iniset -sudo $unit_file "Service" "LimitNOFILE" "65536"
+    iniset -sudo $unit_file "Service" "Environment" "$environment"
+    if [ -n "$stop_cmd" ]; then
+        iniset -sudo $unit_file "Service" "ExecStop" "$stop_cmd"
+    fi
+
+    _start_process $systemd_service
+
+    local testcmd="test -e $OVS_RUNDIR/$service.pid"
+    test_with_retry "$testcmd" "$service did not start" $SERVICE_TIMEOUT 1
+    local service_ctl_file
+    service_ctl_file=$(ls $OVS_RUNDIR | grep $service | grep ctl)
+    if [ -z "$service_ctl_file" ]; then
+        die $LINENO "ctl file for service $service is not present."
+    fi
+    sudo ovs-appctl -t $OVS_RUNDIR/$service_ctl_file vlog/set console:off syslog:info file:info
+}
+
+function clone_repository {
+    local repo=$1
+    local dir=$2
+    local branch=$3
+    # Set ERROR_ON_CLONE to false to avoid the need of having the
+    # repositories like OVN and OVS in the required_projects of the job
+    # definition.
+    ERROR_ON_CLONE=false git_clone $repo $dir $branch
+}
+
+function create_public_bridge {
+    # Create the public bridge that OVN will use
+    sudo ovs-vsctl --may-exist add-br $PUBLIC_BRIDGE -- set bridge $PUBLIC_BRIDGE protocols=OpenFlow13,OpenFlow15
+    sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=$PHYSICAL_NETWORK:$PUBLIC_BRIDGE
+    _configure_public_network_connectivity
+}
+
+function _disable_libvirt_apparmor {
+    if ! sudo aa-status --enabled ; then
+        return 0
+    fi
+    # NOTE(arosen): This is used as a work around to allow newer versions
+    # of libvirt to work with ovs configured ports. See LP#1466631.
+    # requires the apparmor-utils
+    install_package apparmor-utils
+    # disables apparmor for libvirtd
+    sudo aa-complain /etc/apparmor.d/usr.sbin.libvirtd
+}
+
+
+# OVN compilation functions
+# -------------------------
+
+
+# compile_ovn() - Compile OVN from source and load needed modules
+#                 Accepts three parameters:
+#                   - first optional parameter defines prefix for
+#                     ovn compilation
+#                   - second optional parameter defines localstatedir for
+#                     ovn single machine runtime
+function compile_ovn {
+    local prefix=$1
+    local localstatedir=$2
+
+    if [ -n "$prefix" ]; then
+        prefix="--prefix=$prefix"
+    fi
+
+    if [ -n "$localstatedir" ]; then
+        localstatedir="--localstatedir=$localstatedir"
+    fi
+
+    clone_repository $OVN_REPO $DEST/$OVN_REPO_NAME $OVN_BRANCH
+    pushd $DEST/$OVN_REPO_NAME
+
+    if [ ! -f configure ] ; then
+        ./boot.sh
+    fi
+
+    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
+        ./configure --with-ovs-source=$DEST/$OVS_REPO_NAME $prefix $localstatedir
+    fi
+    make -j$(($(nproc) + 1))
+    sudo make install
+    popd
+}
+
+
+# OVN Neutron driver functions
+# ----------------------------
+
+# OVN service sanity check
+function ovn_sanity_check {
+    if is_service_enabled q-agt neutron-agt; then
+        die $LINENO "The q-agt/neutron-agt service must be disabled with OVN."
+    elif is_service_enabled q-l3 neutron-l3; then
+        die $LINENO "The q-l3/neutron-l3 service must be disabled with OVN."
+    elif is_service_enabled q-svc neutron-api && [[ ! $Q_ML2_PLUGIN_MECHANISM_DRIVERS =~ "ovn" ]]; then
+        die $LINENO "OVN needs to be enabled in \$Q_ML2_PLUGIN_MECHANISM_DRIVERS"
+    elif is_service_enabled q-svc neutron-api && [[ ! $Q_ML2_PLUGIN_TYPE_DRIVERS =~ "geneve" ]]; then
+        die $LINENO "Geneve needs to be enabled in \$Q_ML2_PLUGIN_TYPE_DRIVERS to be used with OVN"
+    fi
+}
+
+# install_ovn() - Collect source and prepare
+function install_ovn {
+    echo "Installing OVN and dependent packages"
+
+    # Check the OVN configuration
+    ovn_sanity_check
+
+    # Install tox, used to generate the config (see devstack/override-defaults)
+    pip_install tox
+
+    sudo mkdir -p $OVS_RUNDIR
+    sudo chown $(whoami) $OVS_RUNDIR
+    # NOTE(lucasagomes): To keep things simpler, let's reuse the same
+    # RUNDIR for both OVS and OVN. This way we avoid having to specify the
+    # --db option in the ovn-{n,s}bctl commands while playing with DevStack
+    sudo ln -s $OVS_RUNDIR $OVN_RUNDIR
+
+    if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+        # If OVS is already installed, remove it, because we're about to
+        # re-install it from source.
+        for package in openvswitch openvswitch-switch openvswitch-common; do
+            if is_package_installed $package ; then
+                uninstall_package $package
+            fi
+        done
+
+        remove_ovs_packages
+        sudo rm -f $OVS_RUNDIR/*
+
+        compile_ovs $OVN_BUILD_MODULES
+        if use_new_ovn_repository; then
+            compile_ovn
+        fi
+
+        sudo mkdir -p $OVS_PREFIX/var/log/openvswitch
+        sudo chown $(whoami) $OVS_PREFIX/var/log/openvswitch
+        sudo mkdir -p $OVS_PREFIX/var/log/ovn
+        sudo chown $(whoami) $OVS_PREFIX/var/log/ovn
+    else
+        # Load fixup_ovn_centos
+        source ${TOP_DIR}/tools/fixup_stuff.sh
+        fixup_ovn_centos
+        install_package $(get_packages openvswitch)
+        install_package $(get_packages ovn)
+    fi
+
+    # Ensure that the OVS commands are accessible in the PATH
+    export PATH=$OVS_BINDIR:$PATH
+
+    # Archive log files and create new
+    local log_archive_dir=$LOGDIR/archive
+    mkdir -p $log_archive_dir
+    for logfile in ovs-vswitchd.log ovn-northd.log ovn-controller.log ovn-controller-vtep.log ovs-vtep.log ovsdb-server.log ovsdb-server-nb.log ovsdb-server-sb.log; do
+        if [ -f "$LOGDIR/$logfile" ] ; then
+            mv "$LOGDIR/$logfile"  "$log_archive_dir/$logfile.${CURRENT_LOG_TIME}"
+        fi
+    done
+
+    # Install ovsdbapp from source if requested
+    if use_library_from_git "ovsdbapp"; then
+        git_clone_by_name "ovsdbapp"
+        setup_dev_lib "ovsdbapp"
+    fi
+
+    # Install ovs python module from ovs source.
+    if [[ "$OVN_INSTALL_OVS_PYTHON_MODULE" == "True" ]]; then
+        sudo pip uninstall -y ovs
+        # Clone the OVS repository if it's not yet present
+        clone_repository $OVS_REPO $DEST/$OVS_REPO_NAME $OVS_BRANCH
+        sudo pip install -e $DEST/$OVS_REPO_NAME/python
+    fi
+}
+
+# filter_network_api_extensions() - Remove non-supported API extensions by
+# the OVN driver from the list of enabled API extensions
+function filter_network_api_extensions {
+    SUPPORTED_NETWORK_API_EXTENSIONS=$($PYTHON -c \
+        'from neutron.common.ovn import extensions ;\
+        print(",".join(extensions.ML2_SUPPORTED_API_EXTENSIONS))')
+    SUPPORTED_NETWORK_API_EXTENSIONS=$SUPPORTED_NETWORK_API_EXTENSIONS,$($PYTHON -c \
+        'from neutron.common.ovn import extensions ;\
+        print(",".join(extensions.ML2_SUPPORTED_API_EXTENSIONS_OVN_L3))')
+    if is_service_enabled q-qos neutron-qos ; then
+        SUPPORTED_NETWORK_API_EXTENSIONS="$SUPPORTED_NETWORK_API_EXTENSIONS,qos"
+    fi
+    NETWORK_API_EXTENSIONS=${NETWORK_API_EXTENSIONS:-$SUPPORTED_NETWORK_API_EXTENSIONS}
+    extensions=$(echo $NETWORK_API_EXTENSIONS | tr ', ' '\n' | sort -u)
+    supported_ext=$(echo $SUPPORTED_NETWORK_API_EXTENSIONS | tr ', ' '\n' | sort -u)
+    enabled_ext=$(comm -12 <(echo -e "$extensions") <(echo -e "$supported_ext"))
+    disabled_ext=$(comm -3 <(echo -e "$extensions") <(echo -e "$enabled_ext"))
+
+    # Log a message in case some extensions had to be disabled because
+    # they are not supported by the OVN driver
+    if [ ! -z "$disabled_ext" ]; then
+        _disabled=$(echo $disabled_ext | tr ' ' ',')
+        echo "The folling network API extensions have been disabled because they are not supported by OVN: $_disabled"
+    fi
+
+    # Export the final list of extensions that have been enabled and are
+    # supported by OVN
+    export NETWORK_API_EXTENSIONS=$(echo $enabled_ext | tr ' ' ',')
+}
+
+function configure_ovn_plugin {
+    echo "Configuring Neutron for OVN"
+
+    if is_service_enabled q-svc ; then
+        filter_network_api_extensions
+        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2_type_geneve max_header_size=$OVN_GENEVE_OVERHEAD
+        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_connection="$OVN_NB_REMOTE"
+        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_sb_connection="$OVN_SB_REMOTE"
+        if is_service_enabled tls-proxy; then
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_sb_ca_cert="$INT_CA_DIR/ca-chain.pem"
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_sb_certificate="$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt"
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_sb_private_key="$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key"
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_ca_cert="$INT_CA_DIR/ca-chain.pem"
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_certificate="$INT_CA_DIR/$DEVSTACK_CERT_NAME.crt"
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_private_key="$INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key"
+        fi
+        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn neutron_sync_mode="$OVN_NEUTRON_SYNC_MODE"
+        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_l3_scheduler="$OVN_L3_SCHEDULER"
+        populate_ml2_config /$Q_PLUGIN_CONF_FILE securitygroup enable_security_group="$Q_USE_SECGROUP"
+        inicomment /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver
+
+        if is_service_enabled q-log neutron-log; then
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE network_log rate_limit="$Q_LOG_DRIVER_RATE_LIMIT"
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE network_log burst_limit="$Q_LOG_DRIVER_BURST_LIMIT"
+            inicomment /$Q_PLUGIN_CONF_FILE network_log local_output_log_base="$Q_LOG_DRIVER_LOG_BASE"
+        fi
+
+        if is_service_enabled q-ovn-metadata-agent; then
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
+        else
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=False
+        fi
+
+        if is_service_enabled q-dns neutron-dns ; then
+            iniset $NEUTRON_CONF DEFAULT dns_domain openstackgate.local
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn dns_servers="$OVN_DNS_SERVERS"
+        fi
+
+        iniset $NEUTRON_CONF ovs igmp_snooping_enable $OVN_IGMP_SNOOPING_ENABLE
+    fi
+
+    if is_service_enabled q-dhcp neutron-dhcp ; then
+        iniset $NEUTRON_CONF DEFAULT dhcp_agent_notification True
+    else
+        iniset $NEUTRON_CONF DEFAULT dhcp_agent_notification False
+    fi
+
+    if is_service_enabled n-api-meta ; then
+        if is_service_enabled q-ovn-metadata-agent ; then
+            iniset $NOVA_CONF neutron service_metadata_proxy True
+        fi
+    fi
+}
+
+function configure_ovn {
+    echo "Configuring OVN"
+
+    if [ -z "$OVN_UUID" ] ; then
+        if [ -f $OVS_SYSCONFDIR/system-id.conf ]; then
+            OVN_UUID=$(cat $OVS_SYSCONFDIR/system-id.conf)
+        else
+            OVN_UUID=$(uuidgen)
+            echo $OVN_UUID | sudo tee $OVS_SYSCONFDIR/system-id.conf
+        fi
+    else
+        local ovs_uuid
+        ovs_uuid=$(cat $OVS_SYSCONFDIR/system-id.conf)
+        if [ "$ovs_uuid" != $OVN_UUID ]; then
+            echo $OVN_UUID | sudo tee $OVS_SYSCONFDIR/system-id.conf
+        fi
+    fi
+
+    # Erase the pre-set configurations from packages. DevStack will
+    # configure OVS and OVN accordingly for its use.
+    if [[ "$OVN_BUILD_FROM_SOURCE" == "False" ]] && is_fedora; then
+        sudo truncate -s 0 /etc/openvswitch/default.conf
+        sudo truncate -s 0 /etc/sysconfig/openvswitch
+        sudo truncate -s 0 /etc/sysconfig/ovn
+    fi
+
+    # Metadata
+    if is_service_enabled q-ovn-metadata-agent && is_service_enabled ovn-controller; then
+        sudo install -d -o $STACK_USER $NEUTRON_CONF_DIR
+
+        mkdir -p $NEUTRON_DIR/etc/neutron/plugins/ml2
+        (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)
+
+        cp $NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample $OVN_META_CONF
+        configure_root_helper_options $OVN_META_CONF
+
+        iniset $OVN_META_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
+        iniset $OVN_META_CONF DEFAULT nova_metadata_host $OVN_META_DATA_HOST
+        iniset $OVN_META_CONF DEFAULT metadata_workers $API_WORKERS
+        iniset $OVN_META_CONF DEFAULT state_path $NEUTRON_STATE_PATH
+        iniset $OVN_META_CONF ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
+        iniset $OVN_META_CONF ovn ovn_sb_connection $OVN_SB_REMOTE
+        if is_service_enabled tls-proxy; then
+            iniset $OVN_META_CONF ovn \
+                ovn_sb_ca_cert $INT_CA_DIR/ca-chain.pem
+            iniset $OVN_META_CONF ovn \
+                ovn_sb_certificate $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt
+            iniset $OVN_META_CONF ovn \
+                ovn_sb_private_key $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key
+        fi
+    fi
+}
+
+function init_ovn {
+    # clean up from previous (possibly aborted) runs
+    # create required data files
+
+    # Assumption: this is a dedicated test system and there is nothing important
+    # in the ovn, ovn-nb, or ovs databases.  We're going to trash them and
+    # create new ones on each devstack run.
+
+    _disable_libvirt_apparmor
+    local mkdir_cmd="mkdir -p ${OVN_DATADIR}"
+
+    if [[ "$OVN_BUILD_FROM_SOURCE" == "False" ]]; then
+        mkdir_cmd="sudo ${mkdir_cmd}"
+    fi
+
+    $mkdir_cmd
+    mkdir -p $OVS_DATADIR
+
+    rm -f $OVS_DATADIR/*.db
+    rm -f $OVS_DATADIR/.*.db.~lock~
+    sudo rm -f $OVN_DATADIR/*.db
+    sudo rm -f $OVN_DATADIR/.*.db.~lock~
+    sudo rm -f $OVN_RUNDIR/*.sock
+}
+
+function _start_ovs {
+    echo "Starting OVS"
+    if is_service_enabled ovn-controller ovn-controller-vtep ovn-northd; then
+        # ovsdb-server and ovs-vswitchd are used privately in OVN as openvswitch service names.
+        enable_service ovsdb-server
+        enable_service ovs-vswitchd
+
+        if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+            if [ ! -f $OVS_DATADIR/conf.db ]; then
+                ovsdb-tool create $OVS_DATADIR/conf.db $OVS_SHAREDIR/vswitch.ovsschema
+            fi
+
+            if is_service_enabled ovn-controller-vtep; then
+                if [ ! -f $OVS_DATADIR/vtep.db ]; then
+                    ovsdb-tool create $OVS_DATADIR/vtep.db $OVS_SHAREDIR/vtep.ovsschema
+                fi
+            fi
+
+            local dbcmd="$OVS_SBINDIR/ovsdb-server --remote=punix:$OVS_RUNDIR/db.sock --remote=ptcp:6640:$OVSDB_SERVER_LOCAL_HOST --pidfile --detach --log-file"
+            dbcmd+=" --remote=db:Open_vSwitch,Open_vSwitch,manager_options"
+            if is_service_enabled ovn-controller-vtep; then
+                dbcmd+=" --remote=db:hardware_vtep,Global,managers $OVS_DATADIR/vtep.db"
+            fi
+            dbcmd+=" $OVS_DATADIR/conf.db"
+            _run_process ovsdb-server "$dbcmd" "" "$STACK_GROUP" "root"
+
+            # Note: ovn-controller will create and configure br-int once it is started.
+            # So, no need to create it now because nothing depends on that bridge here.
+            local ovscmd="$OVS_SBINDIR/ovs-vswitchd --log-file --pidfile --detach"
+            _run_process ovs-vswitchd "$ovscmd" "" "$STACK_GROUP" "root"
+        else
+            _start_process "$OVSDB_SERVER_SERVICE"
+            _start_process "$OVS_VSWITCHD_SERVICE"
+        fi
+
+        echo "Configuring OVSDB"
+        if is_service_enabled tls-proxy; then
+            sudo ovs-vsctl --no-wait set-ssl \
+                $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key \
+                $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt \
+                $INT_CA_DIR/ca-chain.pem
+        fi
+
+        sudo ovs-vsctl --no-wait set-manager ptcp:6640:$OVSDB_SERVER_LOCAL_HOST
+        sudo ovs-vsctl --no-wait set open_vswitch . system-type="devstack"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:system-id="$OVN_UUID"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-remote="$OVN_SB_REMOTE"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-bridge="br-int"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-type="geneve"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$TUNNEL_IP"
+        sudo ovs-vsctl --no-wait set open_vswitch . external-ids:hostname=$(hostname)
+        # Select this chassis to host gateway routers
+        if [[ "$ENABLE_CHASSIS_AS_GW" == "True" ]]; then
+            sudo ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-cms-options="enable-chassis-as-gw"
+        fi
+
+        if is_provider_network || [[ $Q_USE_PROVIDERNET_FOR_PUBLIC == "True" ]]; then
+            ovn_base_setup_bridge $OVS_PHYSICAL_BRIDGE
+            sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=${PHYSICAL_NETWORK}:${OVS_PHYSICAL_BRIDGE}
+        fi
+
+        if is_service_enabled ovn-controller-vtep ; then
+            ovn_base_setup_bridge br-v
+            vtep-ctl add-ps br-v
+            vtep-ctl set Physical_Switch br-v tunnel_ips=$TUNNEL_IP
+
+            enable_service ovs-vtep
+            local vtepcmd="$OVS_SCRIPTDIR/ovs-vtep --log-file --pidfile --detach br-v"
+            _run_process ovs-vtep "$vtepcmd" "" "$STACK_GROUP" "root"
+
+            vtep-ctl set-manager tcp:$HOST_IP:6640
+        fi
+    fi
+}
+
+function _start_ovn_services {
+    _start_process "$OVSDB_SERVER_SERVICE"
+    _start_process "$OVS_VSWITCHD_SERVICE"
+
+    if is_service_enabled ovn-northd ; then
+        _start_process "$OVN_NORTHD_SERVICE"
+    fi
+    if is_service_enabled ovn-controller ; then
+        _start_process "$OVN_CONTROLLER_SERVICE"
+    fi
+    if is_service_enabled ovn-controller-vtep ; then
+        _start_process "$OVN_CONTROLLER_VTEP_SERVICE"
+    fi
+    if is_service_enabled ovs-vtep ; then
+        _start_process "devstack@ovs-vtep.service"
+    fi
+    if is_service_enabled q-ovn-metadata-agent; then
+        _start_process "devstack@q-ovn-metadata-agent.service"
+    fi
+}
+
+# start_ovn() - Start running processes, including screen
+function start_ovn {
+    echo "Starting OVN"
+
+    _start_ovs
+
+    local SCRIPTDIR=$OVN_SCRIPTDIR
+    if ! use_new_ovn_repository; then
+        SCRIPTDIR=$OVS_SCRIPTDIR
+    fi
+
+    if is_service_enabled ovn-northd ; then
+        if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+            local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_northd"
+            local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_northd"
+
+            _run_process ovn-northd "$cmd" "$stop_cmd" "$STACK_GROUP" "root"
+        else
+            _start_process "$OVN_NORTHD_SERVICE"
+        fi
+
+        # Wait for the service to be ready
+        # Check for socket and db files for both OVN NB and SB
+        wait_for_sock_file $OVS_RUNDIR/ovnnb_db.sock
+        wait_for_sock_file $OVS_RUNDIR/ovnsb_db.sock
+        wait_for_db_file $OVN_DATADIR/ovnnb_db.db
+        wait_for_db_file $OVN_DATADIR/ovnsb_db.db
+
+        if is_service_enabled tls-proxy; then
+            sudo ovn-nbctl --db=unix:$OVS_RUNDIR/ovnnb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
+            sudo ovn-sbctl --db=unix:$OVS_RUNDIR/ovnsb_db.sock set-ssl $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt $INT_CA_DIR/ca-chain.pem
+        fi
+        sudo ovn-nbctl --db=unix:$OVS_RUNDIR/ovnnb_db.sock set-connection p${OVN_PROTO}:6641:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
+        sudo ovn-sbctl --db=unix:$OVS_RUNDIR/ovnsb_db.sock set-connection p${OVN_PROTO}:6642:$SERVICE_LISTEN_ADDRESS -- set connection . inactivity_probe=60000
+        sudo ovs-appctl -t $OVS_RUNDIR/ovnnb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
+        sudo ovs-appctl -t $OVS_RUNDIR/ovnsb_db.ctl vlog/set console:off syslog:$OVN_DBS_LOG_LEVEL file:$OVN_DBS_LOG_LEVEL
+    fi
+
+    if is_service_enabled ovn-controller ; then
+        if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+            local cmd="/bin/bash $SCRIPTDIR/ovn-ctl --no-monitor start_controller"
+            local stop_cmd="/bin/bash $SCRIPTDIR/ovn-ctl stop_controller"
+
+            _run_process ovn-controller "$cmd" "$stop_cmd" "$STACK_GROUP" "root"
+        else
+            _start_process "$OVN_CONTROLLER_SERVICE"
+        fi
+    fi
+
+    if is_service_enabled ovn-controller-vtep ; then
+        if [[ "$OVN_BUILD_FROM_SOURCE" == "True" ]]; then
+            local cmd="$OVS_BINDIR/ovn-controller-vtep --log-file --pidfile --detach --ovnsb-db=$OVN_SB_REMOTE"
+            _run_process ovn-controller-vtep "$cmd" "" "$STACK_GROUP" "root"
+        else
+            _start_process "$OVN_CONTROLLER_VTEP_SERVICE"
+        fi
+    fi
+
+    if is_service_enabled q-ovn-metadata-agent; then
+        run_process q-ovn-metadata-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_METADATA_BINARY --config-file $OVN_META_CONF"
+        # Format logging
+        setup_logging $OVN_META_CONF
+    fi
+
+    _start_ovn_services
+}
+
+function _stop_ovs_dp {
+    sudo ovs-dpctl dump-dps | sudo xargs -n1 ovs-dpctl del-dp
+    modprobe -q -r vport_geneve vport_vxlan openvswitch || true
+}
+
+function _stop_process {
+    local service=$1
+    echo "Stopping process $service"
+    if $SYSTEMCTL is-enabled $service; then
+        $SYSTEMCTL stop $service
+        $SYSTEMCTL disable $service
+    fi
+}
+
+function stop_ovn {
+    if is_service_enabled q-ovn-metadata-agent; then
+        sudo pkill -9 -f haproxy || :
+        _stop_process "devstack@q-ovn-metadata-agent.service"
+    fi
+    if is_service_enabled ovn-controller-vtep ; then
+        _stop_process "$OVN_CONTROLLER_VTEP_SERVICE"
+    fi
+    if is_service_enabled ovn-controller ; then
+        _stop_process "$OVN_CONTROLLER_SERVICE"
+    fi
+    if is_service_enabled ovn-northd ; then
+        _stop_process "$OVN_NORTHD_SERVICE"
+    fi
+    if is_service_enabled ovs-vtep ; then
+        _stop_process "devstack@ovs-vtep.service"
+    fi
+
+    _stop_process "$OVS_VSWITCHD_SERVICE"
+    _stop_process "$OVSDB_SERVER_SERVICE"
+
+    _stop_ovs_dp
+}
+
+function _cleanup {
+    local path=${1:-$DEST/$OVN_REPO_NAME}
+    pushd $path
+    cd $path
+    sudo make uninstall
+    sudo make distclean
+    popd
+}
+
+# cleanup_ovn() - Remove residual data files, anything left over from previous
+# runs that a clean run would need to clean up
+function cleanup_ovn {
+    local ovn_path=$DEST/$OVN_REPO_NAME
+    local ovs_path=$DEST/$OVS_REPO_NAME
+
+    if [ -d $ovn_path ]; then
+        _cleanup $ovn_path
+    fi
+
+    if [ -d $ovs_path ]; then
+        _cleanup $ovs_path
+    fi
+
+    sudo rm -f $OVN_RUNDIR
+}
@@ -7,6 +7,12 @@
 _XTRACE_NEUTRON_OVS_BASE=$(set +o | grep xtrace)
 set +o xtrace

+# Load devstack ovs compliation and loading functions
+source ${TOP_DIR}/lib/neutron_plugins/ovs_source
+
+# Defaults
+# --------
+
 OVS_BRIDGE=${OVS_BRIDGE:-br-int}
 # OVS recognize default 'system' datapath or 'netdev' for userspace datapath
 OVS_DATAPATH_TYPE=${OVS_DATAPATH_TYPE:-system}
@@ -60,26 +66,33 @@ function _neutron_ovs_base_install_ubuntu_dkms {
 }

 function _neutron_ovs_base_install_agent_packages {
-    # Install deps
-    install_package $(get_packages "openvswitch")
-    if is_ubuntu; then
-        _neutron_ovs_base_install_ubuntu_dkms
-        restart_service openvswitch-switch
-    elif is_fedora; then
-        restart_service openvswitch
-        sudo systemctl enable openvswitch
-    elif is_suse; then
-        if [[ $DISTRO == "sle12" ]] && vercmp "$os_RELEASE" "<" "12.2" ; then
+    if [ "$Q_BUILD_OVS_FROM_GIT" == "True" ]; then
+        remove_ovs_packages
+        compile_ovs False /usr/local /var
+        load_conntrack_gre_module
+        start_new_ovs
+    else
+        # Install deps
+        install_package $(get_packages "openvswitch")
+        if is_ubuntu; then
+            _neutron_ovs_base_install_ubuntu_dkms
            restart_service openvswitch-switch
-        else
-            # workaround for https://bugzilla.suse.com/show_bug.cgi?id=1085971
-            if [[ $DISTRO =~ "tumbleweed" ]]; then
-                sudo sed -i -e "s,^OVS_USER_ID=.*,OVS_USER_ID='root:root'," /etc/sysconfig/openvswitch
+        elif is_fedora; then
+            restart_service openvswitch
+            sudo systemctl enable openvswitch
+        elif is_suse; then
+            if [[ $DISTRO == "sle12" ]] && vercmp "$os_RELEASE" "<" "12.2" ; then
+                restart_service openvswitch-switch
+            else
+                # workaround for https://bugzilla.suse.com/show_bug.cgi?id=1085971
+                if [[ $DISTRO =~ "tumbleweed" ]]; then
+                    sudo sed -i -e "s,^OVS_USER_ID=.*,OVS_USER_ID='root:root'," /etc/sysconfig/openvswitch
+                fi
+                restart_service openvswitch || {
+                    journalctl -xe || :
+                    systemctl status openvswitch
+                }
            fi
-            restart_service openvswitch || {
-                journalctl -xe || :
-                systemctl status openvswitch
-            }
        fi
    fi
 }
@@ -0,0 +1,220 @@
+#!/bin/bash
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Defaults
+# --------
+Q_BUILD_OVS_FROM_GIT=$(trueorfalse False Q_BUILD_OVS_FROM_GIT)
+
+# Set variables for building OVS from source
+OVS_REPO=${OVS_REPO:-https://github.com/openvswitch/ovs.git}
+OVS_REPO_NAME=$(basename ${OVS_REPO} | cut -f1 -d'.')
+OVS_REPO_NAME=${OVS_REPO_NAME:-ovs}
+OVS_BRANCH=${OVS_BRANCH:-0047ca3a0290f1ef954f2c76b31477cf4b9755f5}
+
+# Functions
+
+# load_module() - Load module using modprobe module given by argument and dies
+#                 on failure
+#               - fatal argument is optional and says whether function should
+#                 exit if module can't be loaded
+function load_module {
+    local module=$1
+    local fatal=$2
+
+    if [ "$(trueorfalse True fatal)" == "True" ]; then
+        sudo modprobe $module || (dmesg && die $LINENO "FAILED TO LOAD $module")
+    else
+        sudo modprobe $module || (echo "FAILED TO LOAD $module" && dmesg)
+    fi
+}
+
+# prepare_for_compilation() - Fetch ovs git repository and install packages needed for
+#                             compilation.
+function prepare_for_ovs_compilation {
+    local build_modules=${1:-False}
+    OVS_DIR=$DEST/$OVS_REPO_NAME
+
+    if [ ! -d $OVS_DIR ] ; then
+        # We can't use git_clone here because we want to ignore ERROR_ON_CLONE
+        git_timed clone $OVS_REPO $OVS_DIR
+        cd $OVS_DIR
+        git checkout $OVS_BRANCH
+    else
+        # Even though the directory already exists, call git_clone to update it
+        # if needed based on the RECLONE option
+        git_clone $OVS_REPO $OVS_DIR $OVS_BRANCH
+        cd $OVS_DIR
+    fi
+
+    # TODO: Can you create package list files like you can inside devstack?
+    install_package autoconf automake libtool gcc patch make
+
+    # If build_modules is False, we don't need to install the kernel-*
+    # packages. Just return.
+    if [[ "$build_modules" == "False" ]]; then
+        return
+    fi
+
+    KERNEL_VERSION=`uname -r`
+    if is_fedora ; then
+        # is_fedora covers Fedora, RHEL, CentOS, etc...
+        if [[ "$os_VENDOR" == "Fedora" ]]; then
+            install_package elfutils-libelf-devel
+            KERNEL_VERSION=`echo $KERNEL_VERSION | cut --delimiter='-' --field 1`
+        elif [[ ${KERNEL_VERSION:0:2} != "3." ]]; then
+            # dash is illegal character in rpm version so replace
+            # them with underscore like it is done in the kernel
+            # https://github.com/torvalds/linux/blob/master/scripts/package/mkspec#L25
+            # but only for latest series of the kernel, not 3.x
+
+            KERNEL_VERSION=`echo $KERNEL_VERSION | tr - _`
+        fi
+
+        echo NOTE: if kernel-devel-$KERNEL_VERSION or kernel-headers-$KERNEL_VERSION installation
+        echo failed, please, provide a repository with the package, or yum update / reboot
+        echo your machine to get the latest kernel.
+
+        install_package kernel-devel-$KERNEL_VERSION
+        install_package kernel-headers-$KERNEL_VERSION
+        if is_service_enabled tls-proxy; then
+            install_package openssl-devel
+        fi
+
+    elif is_ubuntu ; then
+        install_package linux-headers-$KERNEL_VERSION
+        if is_service_enabled tls-proxy; then
+            install_package libssl-dev
+        fi
+    fi
+}
+
+# load_ovs_kernel_modules() - load openvswitch kernel module
+function load_ovs_kernel_modules {
+    load_module openvswitch
+    load_module vport-geneve False
+    dmesg | tail
+}
+
+# reload_ovs_kernel_modules() - reload openvswitch kernel module
+function reload_ovs_kernel_modules {
+    set +e
+    ovs_system=$(sudo ovs-dpctl dump-dps | grep ovs-system)
+    if [ -n "$ovs_system" ]; then
+        sudo ovs-dpctl del-dp ovs-system
+    fi
+    set -e
+    sudo modprobe -r vport_geneve
+    sudo modprobe -r openvswitch
+    load_ovs_kernel_modules
+}
+
+# compile_ovs() - Compile OVS from source and load needed modules.
+#                 Accepts two parameters:
+#                   - first one is False by default and means that modules are not built and installed.
+#                   - second optional parameter defines prefix for ovs compilation
+#                   - third optional parameter defines localstatedir for ovs single machine runtime
+#                 Env variables OVS_REPO_NAME, OVS_REPO and OVS_BRANCH must be set
+function compile_ovs {
+    local _pwd=$PWD
+    local build_modules=${1:-False}
+    local prefix=$2
+    local localstatedir=$3
+
+    if [ -n "$prefix" ]; then
+        prefix="--prefix=$prefix"
+    fi
+
+    if [ -n "$localstatedir" ]; then
+        localstatedir="--localstatedir=$localstatedir"
+    fi
+
+    prepare_for_ovs_compilation $build_modules
+
+    KERNEL_VERSION=$(uname -r)
+    major_version=$(echo "${KERNEL_VERSION}" | cut -d '.' -f1)
+    patch_level=$(echo "${KERNEL_VERSION}" | cut -d '.' -f2)
+    if [ "${major_version}" -gt 5 ] || [ "${major_version}" == 5 ] && [ "${patch_level}" -gt 5 ]; then
+        echo "NOTE: KERNEL VERSION is ${KERNEL_VERSION} and OVS doesn't support compiling "
+        echo "Kernel module for version higher than 5.5. Skipping module compilation..."
+        build_modules="False"
+    fi
+
+    if [ ! -f configure ] ; then
+        ./boot.sh
+    fi
+    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
+        if [[ "$build_modules" == "True" ]]; then
+            ./configure $prefix $localstatedir --with-linux=/lib/modules/$(uname -r)/build
+        else
+            ./configure $prefix $localstatedir
+        fi
+    fi
+    make -j$(($(nproc) + 1))
+    sudo make install
+    if [[ "$build_modules" == "True" ]]; then
+        sudo make INSTALL_MOD_DIR=kernel/net/openvswitch modules_install
+    fi
+    reload_ovs_kernel_modules
+
+    cd $_pwd
+}
+
+# action_service - call an action over openvswitch service
+#                  Accepts one parameter that can be either
+#                  'start', 'restart' and 'stop'.
+function action_openvswitch {
+    local action=$1
+
+    if is_ubuntu; then
+        ${action}_service openvswitch-switch
+    elif is_fedora; then
+        ${action}_service openvswitch
+    elif is_suse; then
+        if [[ $DISTRO == "sle12" ]] && [[ $os_RELEASE -lt 12.2 ]]; then
+            ${action}_service openvswitch-switch
+        else
+            ${action}_service openvswitch
+        fi
+    fi
+}
+
+# start_new_ovs() - removes old ovs database, creates a new one and starts ovs
+function start_new_ovs {
+    sudo rm -f /etc/openvswitch/conf.db /etc/openvswitch/.conf.db~lock~
+    sudo /usr/local/share/openvswitch/scripts/ovs-ctl start
+}
+
+# stop_new_ovs() - stops ovs
+function stop_new_ovs {
+    local ovs_ctl='/usr/local/share/openvswitch/scripts/ovs-ctl'
+
+    if [ -x $ovs_ctl ] ; then
+        sudo $ovs_ctl stop
+    fi
+}
+
+# remove_ovs_packages() - removes old ovs packages from the system
+function remove_ovs_packages {
+    for package in openvswitch openvswitch-switch openvswitch-common; do
+        if is_package_installed $package; then
+            uninstall_package $package
+        fi
+    done
+}
+
+
+# load_conntrack_gre_module() - loads nf_conntrack_proto_gre kernel module
+function load_conntrack_gre_module {
+    load_module nf_conntrack_proto_gre False
+}
@@ -101,7 +101,6 @@ SUBNETPOOL_SIZE_V4=${SUBNETPOOL_SIZE_V4:-26}
 SUBNETPOOL_SIZE_V6=${SUBNETPOOL_SIZE_V6:-64}

 default_v4_route_devs=$(ip -4 route | grep ^default | awk '{print $5}')
-die_if_not_set $LINENO default_v4_route_devs "Failure retrieving default IPv4 route devices"

 default_v6_route_devs=$(ip -6 route list match default table all | grep via | awk '{print $5}')

@@ -124,21 +123,7 @@ function _configure_neutron_l3_agent {

    neutron_plugin_configure_l3_agent $Q_L3_CONF_FILE

-    # If we've given a PUBLIC_INTERFACE to take over, then we assume
-    # that we can own the whole thing, and privot it into the OVS
-    # bridge. If we are not, we're probably on a single interface
-    # machine, and we just setup NAT so that fixed guests can get out.
-    if [[ -n "$PUBLIC_INTERFACE" ]]; then
-        _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True False "inet"
-
-        if [[ $(ip -f inet6 a s dev "$PUBLIC_INTERFACE" | grep -c 'global') != 0 ]]; then
-            _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False False "inet6"
-        fi
-    else
-        for d in $default_v4_route_devs; do
-            sudo iptables -t nat -A POSTROUTING -o $d -s $FLOATING_RANGE -j MASQUERADE
-        done
-    fi
+    _configure_public_network_connectivity
 }

 # Explicitly set router id in l3 agent configuration
@@ -162,10 +147,6 @@ function _neutron_get_ext_gw_interface {
 }

 function create_neutron_initial_network {
-    local project_id
-    project_id=$(openstack project list | grep " demo " | get_field 1)
-    die_if_not_set $LINENO project_id "Failure retrieving project_id for demo"
-
    # Allow drivers that need to create an initial network to do so here
    if type -p neutron_plugin_create_initial_network_profile > /dev/null; then
        neutron_plugin_create_initial_network_profile $PHYSICAL_NETWORK
@@ -185,15 +166,15 @@ function create_neutron_initial_network {
    if is_provider_network; then
        die_if_not_set $LINENO PHYSICAL_NETWORK "You must specify the PHYSICAL_NETWORK"
        die_if_not_set $LINENO PROVIDER_NETWORK_TYPE "You must specify the PROVIDER_NETWORK_TYPE"
-        NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create $PHYSICAL_NETWORK --project $project_id --provider-network-type $PROVIDER_NETWORK_TYPE --provider-physical-network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider-segment $SEGMENTATION_ID} --share | grep ' id ' | get_field 2)
-        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK $project_id"
+        NET_ID=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" network create $PHYSICAL_NETWORK --provider-network-type $PROVIDER_NETWORK_TYPE --provider-physical-network "$PHYSICAL_NETWORK" ${SEGMENTATION_ID:+--provider-segment $SEGMENTATION_ID} --share -f value -c id)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PHYSICAL_NETWORK"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            if [ -z $SUBNETPOOL_V4_ID ]; then
                fixed_range_v4=$FIXED_RANGE
            fi
-            SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME $project_id"
+            SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} $PROVIDER_SUBNET_NAME --gateway $NETWORK_GATEWAY ${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} --network $NET_ID ${fixed_range_v4:+--subnet-range $fixed_range_v4} -f value -c id)
+            die_if_not_set $LINENO SUBNET_ID "Failure creating SUBNET_ID for $PROVIDER_SUBNET_NAME"
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
@@ -202,8 +183,8 @@ function create_neutron_initial_network {
            if [ -z $SUBNETPOOL_V6_ID ]; then
                fixed_range_v6=$IPV6_PROVIDER_FIXED_RANGE
            fi
-            IPV6_SUBNET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create --project $project_id --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME $project_id"
+            IPV6_SUBNET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" subnet create --ip-version 6 --gateway $IPV6_PROVIDER_NETWORK_GATEWAY $IPV6_PROVIDER_SUBNET_NAME ${SUBNETPOOL_V6_ID:+--subnet-pool $SUBNETPOOL_V6_ID} --network $NET_ID ${fixed_range_v6:+--subnet-range $fixed_range_v6} -f value -c id)
+            die_if_not_set $LINENO IPV6_SUBNET_ID "Failure creating IPV6_SUBNET_ID for $IPV6_PROVIDER_SUBNET_NAME"
        fi

        if [[ $Q_AGENT == "openvswitch" ]]; then
@@ -212,17 +193,17 @@ function create_neutron_initial_network {
            sudo ip link set $PUBLIC_INTERFACE up
        fi
    else
-        NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create --project $project_id "$PRIVATE_NETWORK_NAME" | grep ' id ' | get_field 2)
-        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME $project_id"
+        NET_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" network create "$PRIVATE_NETWORK_NAME" -f value -c id)
+        die_if_not_set $LINENO NET_ID "Failure creating NET_ID for $PRIVATE_NETWORK_NAME"

        if [[ "$IP_VERSION" =~ 4.* ]]; then
            # Create IPv4 private subnet
-            SUBNET_ID=$(_neutron_create_private_subnet_v4 $project_id)
+            SUBNET_ID=$(_neutron_create_private_subnet_v4)
        fi

        if [[ "$IP_VERSION" =~ .*6 ]]; then
            # Create IPv6 private subnet
-            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6 $project_id)
+            IPV6_SUBNET_ID=$(_neutron_create_private_subnet_v6)
        fi
    fi

@@ -230,12 +211,12 @@ function create_neutron_initial_network {
        # Create a router, and add the private subnet as one of its interfaces
        if [[ "$Q_L3_ROUTER_PER_TENANT" == "True" ]]; then
            # create a tenant-owned router.
-            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create --project $project_id $Q_ROUTER_NAME | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $project_id $Q_ROUTER_NAME"
+            ROUTER_ID=$(openstack --os-cloud devstack --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME"
        else
            # Plugin only supports creating a single router, which should be admin owned.
-            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create $Q_ROUTER_NAME | grep ' id ' | get_field 2)
-            die_if_not_set $LINENO ROUTER_ID "Failure creating ROUTER_ID for $Q_ROUTER_NAME"
+            ROUTER_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router create $Q_ROUTER_NAME -f value -c id)
+            die_if_not_set $LINENO ROUTER_ID "Failure creating router $Q_ROUTER_NAME"
        fi

        EXTERNAL_NETWORK_FLAGS="--external"
@@ -244,9 +225,9 @@ function create_neutron_initial_network {
        fi
        # Create an external network, and a subnet. Configure the external network as router gw
        if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" = "True" ]; then
-            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS --provider-network-type ${PUBLIC_PROVIDERNET_TYPE:-flat} ${PUBLIC_PROVIDERNET_SEGMENTATION_ID:+--provider-segment $PUBLIC_PROVIDERNET_SEGMENTATION_ID} --provider-physical-network ${PUBLIC_PHYSICAL_NETWORK} | grep ' id ' | get_field 2)
+            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS --provider-network-type ${PUBLIC_PROVIDERNET_TYPE:-flat} ${PUBLIC_PROVIDERNET_SEGMENTATION_ID:+--provider-segment $PUBLIC_PROVIDERNET_SEGMENTATION_ID} --provider-physical-network ${PUBLIC_PHYSICAL_NETWORK} -f value -c id)
        else
-            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS | grep ' id ' | get_field 2)
+            EXT_NET_ID=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" network create "$PUBLIC_NETWORK_NAME" $EXTERNAL_NETWORK_FLAGS -f value -c id)
        fi
        die_if_not_set $LINENO EXT_NET_ID "Failure creating EXT_NET_ID for $PUBLIC_NETWORK_NAME"

@@ -264,35 +245,32 @@ function create_neutron_initial_network {

 # Create private IPv4 subnet
 function _neutron_create_private_subnet_v4 {
-    local project_id=$1
    if [ -z $SUBNETPOOL_V4_ID ]; then
        fixed_range_v4=$FIXED_RANGE
    fi
-    local subnet_params="--project $project_id "
-    subnet_params+="--ip-version 4 "
+    local subnet_params="--ip-version 4 "
    if [[ -n "$NETWORK_GATEWAY" ]]; then
        subnet_params+="--gateway $NETWORK_GATEWAY "
    fi
+
    subnet_params+="${SUBNETPOOL_V4_ID:+--subnet-pool $SUBNETPOOL_V4_ID} "
    subnet_params+="${fixed_range_v4:+--subnet-range $fixed_range_v4} "
    subnet_params+="--network $NET_ID $PRIVATE_SUBNET_NAME"
    local subnet_id
-    subnet_id=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create $subnet_params | grep ' id ' | get_field 2)
-    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet for $project_id"
+    subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params -f value -c id)
+    die_if_not_set $LINENO subnet_id "Failure creating private IPv4 subnet"
    echo $subnet_id
 }

 # Create private IPv6 subnet
 function _neutron_create_private_subnet_v6 {
-    local project_id=$1
    die_if_not_set $LINENO IPV6_RA_MODE "IPV6 RA Mode not set"
    die_if_not_set $LINENO IPV6_ADDRESS_MODE "IPV6 Address Mode not set"
    local ipv6_modes="--ipv6-ra-mode $IPV6_RA_MODE --ipv6-address-mode $IPV6_ADDRESS_MODE"
    if [ -z $SUBNETPOOL_V6_ID ]; then
        fixed_range_v6=$FIXED_RANGE_V6
    fi
-    local subnet_params="--project $project_id "
-    subnet_params+="--ip-version 6 "
+    local subnet_params="--ip-version 6 "
    if [[ -n "$IPV6_PRIVATE_NETWORK_GATEWAY" ]]; then
        subnet_params+="--gateway $IPV6_PRIVATE_NETWORK_GATEWAY "
    fi
@@ -300,8 +278,8 @@ function _neutron_create_private_subnet_v6 {
    subnet_params+="${fixed_range_v6:+--subnet-range $fixed_range_v6} "
    subnet_params+="$ipv6_modes --network $NET_ID $IPV6_PRIVATE_SUBNET_NAME "
    local ipv6_subnet_id
-    ipv6_subnet_id=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" subnet create $subnet_params | grep ' id ' | get_field 2)
-    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet for $project_id"
+    ipv6_subnet_id=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" subnet create $subnet_params -f value -c id)
+    die_if_not_set $LINENO ipv6_subnet_id "Failure creating private IPv6 subnet"
    echo $ipv6_subnet_id
 }

@@ -334,7 +312,7 @@ function _neutron_create_public_subnet_v6 {

 # Configure neutron router for IPv4 public access
 function _neutron_configure_router_v4 {
-    openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router add subnet $ROUTER_ID $SUBNET_ID
+    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router add subnet $ROUTER_ID $SUBNET_ID
    # Create a public subnet on the external network
    local id_and_ext_gw_ip
    id_and_ext_gw_ip=$(_neutron_create_public_subnet_v4 $EXT_NET_ID)
@@ -342,10 +320,10 @@ function _neutron_configure_router_v4 {
    ext_gw_ip=$(echo $id_and_ext_gw_ip  | get_field 2)
    PUB_SUBNET_ID=$(echo $id_and_ext_gw_ip | get_field 5)
    # Configure the external network as the default router gateway
-    openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID
+    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID

-    # This logic is specific to using the l3-agent for layer 3
-    if is_service_enabled q-l3 || is_service_enabled neutron-l3;  then
+    # This logic is specific to using OVN or the l3-agent for layer 3
+    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-server) || is_service_enabled q-l3 neutron-l3; then
        # Configure and enable public bridge
        local ext_gw_interface="none"
        if is_neutron_ovs_base_plugin; then
@@ -378,7 +356,7 @@ function _neutron_configure_router_v4 {

 # Configure neutron router for IPv6 public access
 function _neutron_configure_router_v6 {
-    openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router add subnet $ROUTER_ID $IPV6_SUBNET_ID
+    openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router add subnet $ROUTER_ID $IPV6_SUBNET_ID
    # Create a public subnet on the external network
    local ipv6_id_and_ext_gw_ip
    ipv6_id_and_ext_gw_ip=$(_neutron_create_public_subnet_v6 $EXT_NET_ID)
@@ -390,11 +368,11 @@ function _neutron_configure_router_v6 {
    # If the external network has not already been set as the default router
    # gateway when configuring an IPv4 public subnet, do so now
    if [[ "$IP_VERSION" == "6" ]]; then
-        openstack --os-cloud devstack-admin --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID
+        openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router set --external-gateway $EXT_NET_ID $ROUTER_ID
    fi

-    # This logic is specific to using the l3-agent for layer 3
-    if is_service_enabled q-l3 || is_service_enabled neutron-l3; then
+    # This logic is specific to using OVN or the l3-agent for layer 3
+    if ([[ $Q_AGENT == "ovn" ]] && [[ "$OVN_L3_CREATE_PUBLIC_NETWORK" == "True" ]] && is_service_enabled q-svc neutron-server) || is_service_enabled q-l3 neutron-l3; then
        # if the Linux host considers itself to be a router then it will
        # ignore all router advertisements
        # Ensure IPv6 RAs are accepted on interfaces with a default route.
@@ -411,7 +389,13 @@ function _neutron_configure_router_v6 {
        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        # Configure and enable public bridge
        # Override global IPV6_ROUTER_GW_IP with the true value from neutron
-        IPV6_ROUTER_GW_IP=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" port list -c 'Fixed IP Addresses' | grep $ipv6_pub_subnet_id | awk -F'ip_address' '{ print $2 }' | cut -f2 -d\' | tr '\n' ' ')
+        # NOTE(slaweq): when enforce scopes is enabled in Neutron, router's
+        # gateway ports aren't visible in API because such ports don't belongs
+        # to any tenant. Because of that, at least temporary we need to find
+        # IPv6 address of the router's gateway in a bit different way.
+        # It can be reverted when bug
+        # https://bugs.launchpad.net/neutron/+bug/1959332 will be fixed
+        IPV6_ROUTER_GW_IP=$(openstack --os-cloud devstack-admin-demo --os-region "$REGION_NAME" router show $ROUTER_ID -c external_gateway_info -f json | grep -C 1 $ipv6_pub_subnet_id | grep ip_address | awk '{print $2}' | tr -d '"')
        die_if_not_set $LINENO IPV6_ROUTER_GW_IP "Failure retrieving IPV6_ROUTER_GW_IP"

        if is_neutron_ovs_base_plugin; then
@@ -419,8 +403,16 @@ function _neutron_configure_router_v6 {
            ext_gw_interface=$(_neutron_get_ext_gw_interface)
            local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}

-            # Configure interface for public bridge
+            # Configure interface for public bridge by setting the interface
+            # to "up" in case the job is running entirely private network based
+            # testing.
+            sudo ip link set $ext_gw_interface up
            sudo ip -6 addr replace $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
+            # Any IPv6 private subnet that uses the default IPV6 subnet pool
+            # and that is plugged into the default router (Q_ROUTER_NAME) will
+            # be reachable from the devstack node (ex: ipv6-private-subnet).
+            # Some scenario tests (such as octavia-tempest-plugin) rely heavily
+            # on this feature.
            local replace_range=${SUBNETPOOL_PREFIX_V6}
            if [[ -z "${SUBNETPOOL_V6_ID}" ]]; then
                replace_range=${FIXED_RANGE_V6}
@@ -437,3 +429,12 @@ function is_networking_extension_supported {
    EXT_LIST=$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" extension list --network -c Alias -f value)
    [[ $EXT_LIST =~ $extension ]] && return 0
 }
+
+function plugin_agent_add_l3_agent_extension {
+    local l3_agent_extension=$1
+    if [[ -z "$L3_AGENT_EXTENSIONS" ]]; then
+        L3_AGENT_EXTENSIONS=$l3_agent_extension
+    elif [[ ! ,${L3_AGENT_EXTENSIONS}, =~ ,${l3_agent_extension}, ]]; then
+        L3_AGENT_EXTENSIONS+=",$l3_agent_extension"
+    fi
+}
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+function configure_placement_service_plugin {
+    neutron_service_plugin_class_add "placement"
+}
+
+function configure_placement_neutron {
+    iniset $NEUTRON_CONF placement auth_type "$NEUTRON_PLACEMENT_AUTH_TYPE"
+    iniset $NEUTRON_CONF placement auth_url "$KEYSTONE_SERVICE_URI"
+    iniset $NEUTRON_CONF placement username "$NEUTRON_PLACEMENT_USERNAME"
+    iniset $NEUTRON_CONF placement password "$SERVICE_PASSWORD"
+    iniset $NEUTRON_CONF placement user_domain_name "$SERVICE_DOMAIN_NAME"
+    iniset $NEUTRON_CONF placement project_name "$SERVICE_TENANT_NAME"
+    iniset $NEUTRON_CONF placement project_domain_name "$SERVICE_DOMAIN_NAME"
+    iniset $NEUTRON_CONF placement region_name "$REGION_NAME"
+}
+
+function configure_placement_extension {
+    configure_placement_service_plugin
+    configure_placement_neutron
+}
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+function configure_qos_service_plugin {
+    neutron_service_plugin_class_add "qos"
+}
+
+
+function configure_qos_core_plugin {
+    configure_qos_$NEUTRON_CORE_PLUGIN
+}
+
+
+function configure_qos_l2_agent {
+    plugin_agent_add_l2_agent_extension "qos"
+}
+
+
+function configure_qos {
+    configure_qos_service_plugin
+    configure_qos_core_plugin
+    configure_qos_l2_agent
+}
+
+function configure_l3_agent_extension_fip_qos {
+    plugin_agent_add_l3_agent_extension "fip_qos"
+}
+
+function configure_l3_agent_extension_gateway_ip_qos {
+    plugin_agent_add_l3_agent_extension "gateway_ip_qos"
+}
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+function configure_trunk_extension {
+    neutron_service_plugin_class_add "trunk"
+}
@@ -83,6 +83,11 @@ fi
 # services and the compute node
 NOVA_CONSOLE_PROXY_COMPUTE_TLS=${NOVA_CONSOLE_PROXY_COMPUTE_TLS:-False}

+# Validate configuration
+if ! is_service_enabled tls-proxy && [ "$NOVA_CONSOLE_PROXY_COMPUTE_TLS" == "True" ]; then
+    die $LINENO "enabling TLS for the console proxy requires the tls-proxy service"
+fi
+
 # Public facing bits
 NOVA_SERVICE_HOST=${NOVA_SERVICE_HOST:-$SERVICE_HOST}
 NOVA_SERVICE_PORT=${NOVA_SERVICE_PORT:-8774}
@@ -96,30 +101,12 @@ NOVA_ENABLE_CACHE=${NOVA_ENABLE_CACHE:-True}
 # NOTE: Set ``FORCE_CONFIG_DRIVE="False"`` to turn OFF config drive
 FORCE_CONFIG_DRIVE=${FORCE_CONFIG_DRIVE:-"False"}

-# Nova supports pluggable schedulers.  The default ``FilterScheduler``
-# should work in most cases.
-SCHEDULER=${SCHEDULER:-filter_scheduler}
-
 # The following NOVA_FILTERS contains SameHostFilter and DifferentHostFilter with
 # the default filters.
-NOVA_FILTERS="AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter"
+NOVA_FILTERS="ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter"

 QEMU_CONF=/etc/libvirt/qemu.conf

-# Set default defaults here as some hypervisor drivers override these
-PUBLIC_INTERFACE_DEFAULT=br100
-# Set ``GUEST_INTERFACE_DEFAULT`` to some interface on the box so that
-# the default isn't completely crazy. This will match ``eth*``, ``em*``, or
-# the new ``p*`` interfaces, then basically picks the first
-# alphabetically. It's probably wrong, however it's less wrong than
-# always using ``eth0`` which doesn't exist on new Linux distros at all.
-GUEST_INTERFACE_DEFAULT=$(ip link \
-    | grep 'state UP' \
-    | awk '{print $2}' \
-    | sed 's/://' \
-    | grep ^[ep] \
-    | head -1)
-
 # ``NOVA_VNC_ENABLED`` can be used to forcibly enable VNC configuration.
 # In multi-node setups allows compute hosts to not run ``n-novnc``.
 NOVA_VNC_ENABLED=$(trueorfalse False NOVA_VNC_ENABLED)
@@ -139,7 +126,7 @@ fi
 # ``NOVA_USE_SERVICE_TOKEN`` is a mode where service token is passed along with
 # user token while communicating to external RESP API's like Neutron, Cinder
 # and Glance.
-NOVA_USE_SERVICE_TOKEN=$(trueorfalse False NOVA_USE_SERVICE_TOKEN)
+NOVA_USE_SERVICE_TOKEN=$(trueorfalse True NOVA_USE_SERVICE_TOKEN)

 # ``NOVA_ALLOW_MOVE_TO_SAME_HOST`` can be set to False in multi node DevStack,
 # where there are at least two nova-computes.
@@ -158,6 +145,9 @@ NOVA_NOTIFICATION_FORMAT=${NOVA_NOTIFICATION_FORMAT:-unversioned}
 # image in devstack is CirrOS.
 NOVA_SHUTDOWN_TIMEOUT=${NOVA_SHUTDOWN_TIMEOUT:-0}

+# Whether to use Keystone unified limits instead of legacy quota limits.
+NOVA_USE_UNIFIED_LIMITS=$(trueorfalse False NOVA_USE_UNIFIED_LIMITS)
+
 # Functions
 # ---------

@@ -232,6 +222,10 @@ function cleanup_nova {
    stop_process "n-api-meta"
    remove_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI"
    remove_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI"
+
+    if [[ "$NOVA_BACKEND" == "LVM" ]]; then
+        clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME
+    fi
 }

 # configure_nova() - Set config files, create data dirs, etc
@@ -259,6 +253,8 @@ function configure_nova {
                if [ ! -e /dev/kvm ]; then
                    echo "WARNING: Switching to QEMU"
                    LIBVIRT_TYPE=qemu
+                    LIBVIRT_CPU_MODE=custom
+                    LIBVIRT_CPU_MODEL=Nehalem
                    if which selinuxenabled >/dev/null 2>&1 && selinuxenabled; then
                        # https://bugzilla.redhat.com/show_bug.cgi?id=753589
                        sudo setsebool virt_use_execmem on
@@ -296,15 +292,8 @@ function configure_nova {
            fi
        fi

-        if is_fedora && [[ $DISTRO =~ f[0-9][0-9] ]]; then
-            # There is an iscsi-initiator bug where it inserts
-            # different whitespace that causes a bunch of output
-            # matching to fail.  We have not been able to get
-            # fixed, yet :/  Exists in fedora 29 & 30 at least
-            #  https://bugzilla.redhat.com/show_bug.cgi?id=1676365
-            sudo dnf copr enable -y iwienand/iscsi-initiator-utils
-            sudo dnf update -y
-        fi
+        # Ensure each compute host uses a unique iSCSI initiator
+        echo InitiatorName=$(iscsi-iname) | sudo tee /etc/iscsi/initiatorname.iscsi

        if [[ ${ISCSID_DEBUG} == "True" ]]; then
            # Install an override that starts iscsid with debugging
@@ -319,6 +308,10 @@ EOF
            sudo systemctl daemon-reload
        fi

+        # set chap algorithms.  The default chap_algorithm is md5 which will
+        # not work under FIPS.
+        iniset -sudo /etc/iscsi/iscsid.conf DEFAULT "node.session.auth.chap_algs" "SHA3-256,SHA256"
+
        # ensure that iscsid is started, even when disabled by default
        restart_service iscsid
    fi
@@ -386,6 +379,13 @@ function create_nova_accounts {
            "http://$SERVICE_HOST:$S3_SERVICE_PORT" \
            "http://$SERVICE_HOST:$S3_SERVICE_PORT"
    fi
+
+    # Unified limits
+    if is_service_enabled n-api; then
+        if [[ "$NOVA_USE_UNIFIED_LIMITS" = True ]]; then
+            configure_nova_unified_limits
+        fi
+    fi
 }

 # create_nova_conf() - Create a new nova.conf file
@@ -401,11 +401,8 @@ function create_nova_conf {
    fi
    iniset $NOVA_CONF wsgi api_paste_config "$NOVA_API_PASTE_INI"
    iniset $NOVA_CONF DEFAULT rootwrap_config "$NOVA_CONF_DIR/rootwrap.conf"
-    iniset $NOVA_CONF scheduler driver "$SCHEDULER"
    iniset $NOVA_CONF filter_scheduler enabled_filters "$NOVA_FILTERS"
-    if [[ $SCHEDULER == "filter_scheduler" ]]; then
-        iniset $NOVA_CONF scheduler workers "$API_WORKERS"
-    fi
+    iniset $NOVA_CONF scheduler workers "$API_WORKERS"
    iniset $NOVA_CONF neutron default_floating_pool "$PUBLIC_NETWORK_NAME"
    if [[ $SERVICE_IP_VERSION == 6 ]]; then
        iniset $NOVA_CONF DEFAULT my_ip "$HOST_IPV6"
@@ -468,11 +465,7 @@ function create_nova_conf {
    fi

    if is_service_enabled cinder; then
-        if is_service_enabled tls-proxy; then
-            CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST}
-            CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776}
-            iniset $NOVA_CONF cinder cafile $SSL_BUNDLE_FILE
-        fi
+        configure_cinder_access
    fi

    if [ -n "$NOVA_STATE_PATH" ]; then
@@ -490,7 +483,8 @@ function create_nova_conf {
    fi

    # nova defaults to genisoimage but only mkisofs is available for 15.0+
-    if is_suse; then
+    # rhel provides mkisofs symlink to genisoimage or xorriso appropiately
+    if is_suse || is_fedora; then
        iniset $NOVA_CONF DEFAULT mkisofs_cmd /usr/bin/mkisofs
    fi

@@ -499,8 +493,13 @@ function create_nova_conf {

    iniset $NOVA_CONF upgrade_levels compute "auto"

-    write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute"
-    write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}"
+    if is_service_enabled n-api; then
+        write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute"
+    fi
+
+    if is_service_enabled n-api-meta; then
+        write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}"
+    fi

    if is_service_enabled ceilometer; then
        iniset $NOVA_CONF DEFAULT instance_usage_audit "True"
@@ -520,8 +519,6 @@ function create_nova_conf {
    # don't let the conductor get out of control now that we're using a pure python db driver
    iniset $NOVA_CONF conductor workers "$API_WORKERS"

-    iniset $NOVA_CONF cinder os_region_name "$REGION_NAME"
-
    if is_service_enabled tls-proxy; then
        iniset $NOVA_CONF DEFAULT glance_protocol https
        iniset $NOVA_CONF oslo_middleware enable_proxy_headers_parsing True
@@ -593,6 +590,29 @@ function configure_placement_nova_compute {
    iniset $conf placement region_name "$REGION_NAME"
 }

+# Configure access to cinder.
+function configure_cinder_access {
+    iniset $NOVA_CONF cinder os_region_name "$REGION_NAME"
+    iniset $NOVA_CONF cinder auth_type "password"
+    iniset $NOVA_CONF cinder auth_url "$KEYSTONE_SERVICE_URI"
+    # NOTE(mriedem): This looks a bit weird but we use the nova user here
+    # since it has the admin role and the cinder user does not. This is
+    # similar to using the nova user in init_nova_service_user_conf. We need
+    # to use a user with the admin role for background tasks in nova to
+    # be able to GET block-storage API resources owned by another project
+    # since cinder has low-level "is_admin" checks in its DB API.
+    iniset $NOVA_CONF cinder username nova
+    iniset $NOVA_CONF cinder password "$SERVICE_PASSWORD"
+    iniset $NOVA_CONF cinder user_domain_name "$SERVICE_DOMAIN_NAME"
+    iniset $NOVA_CONF cinder project_name "$SERVICE_TENANT_NAME"
+    iniset $NOVA_CONF cinder project_domain_name "$SERVICE_DOMAIN_NAME"
+    if is_service_enabled tls-proxy; then
+        CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST}
+        CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776}
+        iniset $NOVA_CONF cinder cafile $SSL_BUNDLE_FILE
+    fi
+}
+
 function configure_console_compute {
    # If we are running multiple cells (and thus multiple console proxies) on a
    # single host, we offset the ports to avoid collisions.  We need to
@@ -600,10 +620,10 @@ function configure_console_compute {
    # can use the NOVA_CPU_CELL variable to know which cell we are for
    # calculating the offset.
    # Stagger the offset based on the total number of possible console proxies
-    # (novnc, xvpvnc, spice, serial) so that their ports will not collide if
+    # (novnc, spice, serial) so that their ports will not collide if
    # all are enabled.
    local offset
-    offset=$(((NOVA_CPU_CELL - 1) * 4))
+    offset=$(((NOVA_CPU_CELL - 1) * 3))

    # Use the host IP instead of the service host because for multi-node, the
    # service host will be the controller only.
@@ -611,7 +631,7 @@ function configure_console_compute {
    default_proxyclient_addr=$(iniget $NOVA_CPU_CONF DEFAULT my_ip)

    # All nova-compute workers need to know the vnc configuration options
-    # These settings don't hurt anything if n-xvnc and n-novnc are disabled
+    # These settings don't hurt anything if n-novnc is disabled
    if is_service_enabled n-cpu; then
        if [ "$NOVNC_FROM_PACKAGE" == "True" ]; then
            # Use the old URL when installing novnc packages.
@@ -624,13 +644,11 @@ function configure_console_compute {
            NOVNCPROXY_URL=${NOVNCPROXY_URL:-"http://$SERVICE_HOST:$((6080 + offset))/vnc_lite.html"}
        fi
        iniset $NOVA_CPU_CONF vnc novncproxy_base_url "$NOVNCPROXY_URL"
-        XVPVNCPROXY_URL=${XVPVNCPROXY_URL:-"http://$SERVICE_HOST:$((6081 + offset))/console"}
-        iniset $NOVA_CPU_CONF vnc xvpvncproxy_base_url "$XVPVNCPROXY_URL"
-        SPICEHTML5PROXY_URL=${SPICEHTML5PROXY_URL:-"http://$SERVICE_HOST:$((6082 + offset))/spice_auto.html"}
+        SPICEHTML5PROXY_URL=${SPICEHTML5PROXY_URL:-"http://$SERVICE_HOST:$((6081 + offset))/spice_auto.html"}
        iniset $NOVA_CPU_CONF spice html5proxy_base_url "$SPICEHTML5PROXY_URL"
    fi

-    if is_service_enabled n-novnc || is_service_enabled n-xvnc || [ "$NOVA_VNC_ENABLED" != False ]; then
+    if is_service_enabled n-novnc || [ "$NOVA_VNC_ENABLED" != False ]; then
        # Address on which instance vncservers will listen on compute hosts.
        # For multi-host, this should be the management ip of the compute host.
        VNCSERVER_LISTEN=${VNCSERVER_LISTEN:-$NOVA_SERVICE_LISTEN_ADDRESS}
@@ -653,7 +671,7 @@ function configure_console_compute {

    if is_service_enabled n-sproxy; then
        iniset $NOVA_CPU_CONF serial_console enabled True
-        iniset $NOVA_CPU_CONF serial_console base_url "ws://$SERVICE_HOST:$((6083 + offset))/"
+        iniset $NOVA_CPU_CONF serial_console base_url "ws://$SERVICE_HOST:$((6082 + offset))/"
    fi
 }

@@ -662,15 +680,13 @@ function configure_console_proxies {
    local conf=${1:-$NOVA_CONF}
    local offset=${2:-0}
    # Stagger the offset based on the total number of possible console proxies
-    # (novnc, xvpvnc, spice, serial) so that their ports will not collide if
+    # (novnc, spice, serial) so that their ports will not collide if
    # all are enabled.
-    offset=$((offset * 4))
+    offset=$((offset * 3))

-    if is_service_enabled n-novnc || is_service_enabled n-xvnc || [ "$NOVA_VNC_ENABLED" != False ]; then
+    if is_service_enabled n-novnc || [ "$NOVA_VNC_ENABLED" != False ]; then
        iniset $conf vnc novncproxy_host "$NOVA_SERVICE_LISTEN_ADDRESS"
        iniset $conf vnc novncproxy_port $((6080 + offset))
-        iniset $conf vnc xvpvncproxy_host "$NOVA_SERVICE_LISTEN_ADDRESS"
-        iniset $conf vnc xvpvncproxy_port $((6081 + offset))

        if is_nova_console_proxy_compute_tls_enabled ; then
            iniset $conf vnc auth_schemes "vencrypt"
@@ -702,15 +718,62 @@ function configure_console_proxies {

    if is_service_enabled n-spice; then
        iniset $conf spice html5proxy_host "$NOVA_SERVICE_LISTEN_ADDRESS"
-        iniset $conf spice html5proxy_port $((6082 + offset))
+        iniset $conf spice html5proxy_port $((6081 + offset))
    fi

    if is_service_enabled n-sproxy; then
        iniset $conf serial_console serialproxy_host "$NOVA_SERVICE_LISTEN_ADDRESS"
-        iniset $conf serial_console serialproxy_port $((6083 + offset))
+        iniset $conf serial_console serialproxy_port $((6082 + offset))
    fi
 }

+function configure_nova_unified_limits {
+    # Registered limit resources in keystone are system-specific resources.
+    # Make sure we use a system-scoped token to interact with this API.
+
+    # Default limits here mirror the legacy config-based default values.
+    # Note: disk quota is new in nova as of unified limits.
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 10 --region $REGION_NAME servers
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 20 --region $REGION_NAME class:VCPU
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit $((50 * 1024)) --region $REGION_NAME class:MEMORY_MB
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 20 --region $REGION_NAME class:DISK_GB
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 128 --region $REGION_NAME server_metadata_items
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 5 --region $REGION_NAME server_injected_files
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 10240 --region $REGION_NAME server_injected_file_content_bytes
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 255 --region $REGION_NAME server_injected_file_path_bytes
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 100 --region $REGION_NAME server_key_pairs
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 10 --region $REGION_NAME server_groups
+    openstack --os-cloud devstack-system-admin registered limit create \
+        --service nova --default-limit 10 --region $REGION_NAME server_group_members
+
+    # Tell nova to use these limits
+    iniset $NOVA_CONF quota driver "nova.quota.UnifiedLimitsDriver"
+
+    # Configure oslo_limit so it can talk to keystone
+    iniset $NOVA_CONF oslo_limit user_domain_name $SERVICE_DOMAIN_NAME
+    iniset $NOVA_CONF oslo_limit password $SERVICE_PASSWORD
+    iniset $NOVA_CONF oslo_limit username nova
+    iniset $NOVA_CONF oslo_limit auth_type password
+    iniset $NOVA_CONF oslo_limit auth_url $KEYSTONE_SERVICE_URI
+    iniset $NOVA_CONF oslo_limit system_scope all
+    iniset $NOVA_CONF oslo_limit endpoint_id \
+           $(openstack endpoint list --service nova -f value -c ID)
+
+    # Allow the nova service user to read quotas
+    openstack --os-cloud devstack-system-admin role add --user nova \
+        --user-domain $SERVICE_DOMAIN_NAME --system all reader
+}
+
 function init_nova_service_user_conf {
    iniset $NOVA_CONF service_user send_service_user_token True
    iniset $NOVA_CONF service_user auth_type password
@@ -734,30 +797,50 @@ function create_nova_keys_dir {
    sudo install -d -o $STACK_USER ${NOVA_STATE_PATH} ${NOVA_STATE_PATH}/keys
 }

+function init_nova_db {
+    local dbname="$1"
+    local conffile="$2"
+    recreate_database $dbname
+    $NOVA_BIN_DIR/nova-manage --config-file $conffile db sync --local_cell
+}
+
 # init_nova() - Initialize databases, etc.
 function init_nova {
    # All nova components talk to a central database.
    # Only do this step once on the API node for an entire cluster.
    if is_service_enabled $DATABASE_BACKENDS && is_service_enabled n-api; then
+        # (Re)create nova databases
+        if [[ "$CELLSV2_SETUP" == "singleconductor" ]]; then
+            # If we are doing singleconductor mode, we have some strange
+            # interdependencies. in that the main config refers to cell1
+            # instead of cell0. In that case, just make sure the cell0 database
+            # is created before we need it below, but don't db_sync it until
+            # after the cellN databases are there.
+            recreate_database nova_cell0
+        else
+            async_run nova-cell-0 init_nova_db nova_cell0 $NOVA_CONF
+        fi
+
+        for i in $(seq 1 $NOVA_NUM_CELLS); do
+            async_run nova-cell-$i init_nova_db nova_cell${i} $(conductor_conf $i)
+        done
+
        recreate_database $NOVA_API_DB
        $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF api_db sync

-        recreate_database nova_cell0
-
        # map_cell0 will create the cell mapping record in the nova_api DB so
-        # this needs to come after the api_db sync happens. We also want to run
-        # this before the db sync below since that will migrate both the nova
-        # and nova_cell0 databases.
+        # this needs to come after the api_db sync happens.
        $NOVA_BIN_DIR/nova-manage cell_v2 map_cell0 --database_connection `database_connection_url nova_cell0`

-        # (Re)create nova databases
-        for i in $(seq 1 $NOVA_NUM_CELLS); do
-            recreate_database nova_cell${i}
-            $NOVA_BIN_DIR/nova-manage --config-file $(conductor_conf $i) db sync --local_cell
+        # Wait for DBs to finish from above
+        for i in $(seq 0 $NOVA_NUM_CELLS); do
+            async_wait nova-cell-$i
        done

-        # Migrate nova and nova_cell0 databases.
-        $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db sync
+        if [[ "$CELLSV2_SETUP" == "singleconductor" ]]; then
+            # We didn't db sync cell0 above, so run it now
+            $NOVA_BIN_DIR/nova-manage --config-file $NOVA_CONF db sync
+        fi

        # Run online migrations on the new databases
        # Needed for flavor conversion
@@ -805,7 +888,7 @@ function install_nova {
            NOVNC_WEB_DIR=/usr/share/novnc
            install_package novnc
        else
-            NOVNC_WEB_DIR=$DEST/noVNC
+            NOVNC_WEB_DIR=$DEST/novnc
            git_clone $NOVNC_REPO $NOVNC_WEB_DIR $NOVNC_BRANCH
        fi
    fi
@@ -850,7 +933,7 @@ function start_nova_api {
            start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT
        fi
    else
-        run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF"
+        run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF"
        nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/
    fi

@@ -899,6 +982,20 @@ function start_nova_compute {
    # by the compute process.
    configure_console_compute

+    # Set rebuild timeout longer for BFV instances because we likely have
+    # slower disk than expected. Default is 20s/GB
+    iniset $NOVA_CPU_CONF DEFAULT reimage_timeout_per_gb 60
+
+    # Configure the OVSDB connection for os-vif
+    if [ -n "$OVSDB_SERVER_LOCAL_HOST" ]; then
+        iniset $NOVA_CPU_CONF os_vif_ovs ovsdb_connection "tcp:$OVSDB_SERVER_LOCAL_HOST:6640"
+    fi
+
+    # Workaround bug #1939108
+    if [[ "$VIRT_DRIVER" == "libvirt" && "$LIBVIRT_TYPE" == "qemu" ]]; then
+        iniset $NOVA_CPU_CONF workarounds libvirt_disable_apic True
+    fi
+
    if [[ "$VIRT_DRIVER" = 'libvirt' ]]; then
        # The group **$LIBVIRT_GROUP** is added to the current user in this script.
        # ``sg`` is used in run_process to execute nova-compute as a member of the
@@ -934,14 +1031,13 @@ function start_nova_rest {
    local old_path=$PATH
    export PATH=$NOVA_BIN_DIR:$PATH

-    local api_cell_conf=$NOVA_CONF
    local compute_cell_conf=$NOVA_CONF

    run_process n-sch "$NOVA_BIN_DIR/nova-scheduler --config-file $compute_cell_conf"
    if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then
        run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf"
    else
-        run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF"
+        run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF"
    fi

    export PATH=$old_path
@@ -949,7 +1045,7 @@ function start_nova_rest {

 function enable_nova_console_proxies {
    for i in $(seq 1 $NOVA_NUM_CELLS); do
-        for srv in n-novnc n-xvnc n-spice n-sproxy; do
+        for srv in n-novnc n-spice n-sproxy; do
            if is_service_enabled $srv; then
                enable_service ${srv}-cell${i}
            fi
@@ -967,7 +1063,6 @@ function start_nova_console_proxies {
    # console proxies run globally for singleconductor, else they run per cell
    if [[ "${CELLSV2_SETUP}" == "singleconductor" ]]; then
        run_process n-novnc "$NOVA_BIN_DIR/nova-novncproxy --config-file $api_cell_conf --web $NOVNC_WEB_DIR"
-        run_process n-xvnc "$NOVA_BIN_DIR/nova-xvpvncproxy --config-file $api_cell_conf"
        run_process n-spice "$NOVA_BIN_DIR/nova-spicehtml5proxy --config-file $api_cell_conf --web $SPICE_WEB_DIR"
        run_process n-sproxy "$NOVA_BIN_DIR/nova-serialproxy --config-file $api_cell_conf"
    else
@@ -976,7 +1071,6 @@ function start_nova_console_proxies {
            local conf
            conf=$(conductor_conf $i)
            run_process n-novnc-cell${i} "$NOVA_BIN_DIR/nova-novncproxy --config-file $conf --web $NOVNC_WEB_DIR"
-            run_process n-xvnc-cell${i} "$NOVA_BIN_DIR/nova-xvpvncproxy --config-file $conf"
            run_process n-spice-cell${i} "$NOVA_BIN_DIR/nova-spicehtml5proxy --config-file $conf --web $SPICE_WEB_DIR"
            run_process n-sproxy-cell${i} "$NOVA_BIN_DIR/nova-serialproxy --config-file $conf"
        done
@@ -1021,14 +1115,6 @@ function is_nova_ready {
    # happen between here and the script ending. However, in multinode
    # tests this can very often not be the case. So ensure that the
    # compute is up before we move on.
-
-    # TODO(sdague): honestly, this probably should be a plug point for
-    # an external system.
-    if [[ "$VIRT_DRIVER" == 'xenserver' ]]; then
-        # xenserver encodes information in the hostname of the compute
-        # because of the dom0/domU split. Just ignore for now.
-        return
-    fi
    wait_for_compute $NOVA_READY_TIMEOUT
 }

@@ -1067,13 +1153,13 @@ function stop_nova_rest {

 function stop_nova_console_proxies {
    if [[ "${CELLSV2_SETUP}" == "singleconductor" ]]; then
-        for srv in n-novnc n-xvnc n-spice n-sproxy; do
+        for srv in n-novnc n-spice n-sproxy; do
            stop_process $srv
        done
    else
        enable_nova_console_proxies
        for i in $(seq 1 $NOVA_NUM_CELLS); do
-            for srv in n-novnc n-xvnc n-spice n-sproxy; do
+            for srv in n-novnc n-spice n-sproxy; do
                stop_process ${srv}-cell${i}
            done
        done
@@ -1107,19 +1193,19 @@ function create_flavors {
    if is_service_enabled n-api; then
        if ! openstack --os-region-name="$REGION_NAME" flavor list | grep -q ds512M; then
            # Note that danms hates these flavors and apologizes for sdague
-            openstack --os-region-name="$REGION_NAME" flavor create --id c1 --ram 256 --disk 1 --vcpus 1 cirros256
-            openstack --os-region-name="$REGION_NAME" flavor create --id d1 --ram 512 --disk 5 --vcpus 1 ds512M
-            openstack --os-region-name="$REGION_NAME" flavor create --id d2 --ram 1024 --disk 10 --vcpus 1 ds1G
-            openstack --os-region-name="$REGION_NAME" flavor create --id d3 --ram 2048 --disk 10 --vcpus 2 ds2G
-            openstack --os-region-name="$REGION_NAME" flavor create --id d4 --ram 4096 --disk 20 --vcpus 4 ds4G
+            openstack --os-region-name="$REGION_NAME" flavor create --id c1 --ram 256 --disk 1 --vcpus 1 --property hw_rng:allowed=True cirros256
+            openstack --os-region-name="$REGION_NAME" flavor create --id d1 --ram 512 --disk 5 --vcpus 1 --property hw_rng:allowed=True ds512M
+            openstack --os-region-name="$REGION_NAME" flavor create --id d2 --ram 1024 --disk 10 --vcpus 1 --property hw_rng:allowed=True ds1G
+            openstack --os-region-name="$REGION_NAME" flavor create --id d3 --ram 2048 --disk 10 --vcpus 2 --property hw_rng:allowed=True ds2G
+            openstack --os-region-name="$REGION_NAME" flavor create --id d4 --ram 4096 --disk 20 --vcpus 4 --property hw_rng:allowed=True ds4G
        fi

        if ! openstack --os-region-name="$REGION_NAME" flavor list | grep -q m1.tiny; then
-            openstack --os-region-name="$REGION_NAME" flavor create --id 1 --ram 512 --disk 1 --vcpus 1 m1.tiny
-            openstack --os-region-name="$REGION_NAME" flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 m1.small
-            openstack --os-region-name="$REGION_NAME" flavor create --id 3 --ram 4096 --disk 40 --vcpus 2 m1.medium
-            openstack --os-region-name="$REGION_NAME" flavor create --id 4 --ram 8192 --disk 80 --vcpus 4 m1.large
-            openstack --os-region-name="$REGION_NAME" flavor create --id 5 --ram 16384 --disk 160 --vcpus 8 m1.xlarge
+            openstack --os-region-name="$REGION_NAME" flavor create --id 1 --ram 512 --disk 1 --vcpus 1 --property hw_rng:allowed=True m1.tiny
+            openstack --os-region-name="$REGION_NAME" flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 --property hw_rng:allowed=True m1.small
+            openstack --os-region-name="$REGION_NAME" flavor create --id 3 --ram 4096 --disk 40 --vcpus 2 --property hw_rng:allowed=True m1.medium
+            openstack --os-region-name="$REGION_NAME" flavor create --id 4 --ram 8192 --disk 80 --vcpus 4 --property hw_rng:allowed=True m1.large
+            openstack --os-region-name="$REGION_NAME" flavor create --id 5 --ram 16384 --disk 160 --vcpus 8 --property hw_rng:allowed=True m1.xlarge
        fi
    fi
 }
@@ -24,17 +24,14 @@ DEBUG_LIBVIRT=$(trueorfalse True DEBUG_LIBVIRT)
 # Currently fairly specific to OpenStackCI hosts
 DEBUG_LIBVIRT_COREDUMPS=$(trueorfalse False DEBUG_LIBVIRT_COREDUMPS)

-# Only Xenial is left with libvirt-bin.  Everywhere else is libvirtd
-if is_ubuntu && [ ${DISTRO} == "xenial" ]; then
-    LIBVIRT_DAEMON=libvirt-bin
-else
-    LIBVIRT_DAEMON=libvirtd
-fi
+# Enable the Fedora Virtualization Preview Copr repo that provides the latest
+# rawhide builds of QEMU, Libvirt and other virt tools.
+ENABLE_FEDORA_VIRT_PREVIEW_REPO=$(trueorfalse False ENABLE_FEDORA_VIRT_PREVIEW_REPO)

 # Enable coredumps for libvirt
 #  Bug: https://bugs.launchpad.net/nova/+bug/1643911
 function _enable_coredump {
-    local confdir=/etc/systemd/system/${LIBVIRT_DAEMON}.service.d
+    local confdir=/etc/systemd/system/libvirtd.service.d
    local conffile=${confdir}/coredump.conf

    # Create a coredump directory, and instruct the kernel to save to
@@ -59,29 +56,42 @@ EOF

 # Installs required distro-specific libvirt packages.
 function install_libvirt {
+    # NOTE(yoctozepto): The common consensus [1] is that libvirt-python should
+    # be installed from distro packages. However, various projects might be
+    # trying to ensure it is installed using pip AND use upper-constraints
+    # with that, causing pip to try to upgrade it and to fail.
+    # The following line removes libvirt-python from upper-constraints and
+    # avoids the situation described above. Now only if installed packages
+    # explicitly depend on a newer (or, in general, incompatible) libvirt-python
+    # version, will pip try to reinstall it.
+    # [1] https://review.opendev.org/c/openstack/devstack/+/798514
+    $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
+            $REQUIREMENTS_DIR/upper-constraints.txt -- libvirt-python

    if is_ubuntu; then
-        install_package qemu-system
-        if [[ ${DISTRO} == "xenial" ]]; then
-            install_package libvirt-bin libvirt-dev
-        else
-            install_package libvirt-clients libvirt-daemon-system libvirt-dev
+        install_package qemu-system libvirt-clients libvirt-daemon-system libvirt-dev python3-libvirt
+        if is_arch "aarch64"; then
+            install_package qemu-efi
        fi
-        # uninstall in case the libvirt version changed
-        pip_uninstall libvirt-python
-        pip_install_gr libvirt-python
        #pip_install_gr <there-si-no-guestfs-in-pypi>
    elif is_fedora || is_suse; then

+        # Optionally enable the virt-preview repo when on Fedora
+        if [[ $DISTRO =~ f[0-9][0-9] ]] && [[ ${ENABLE_FEDORA_VIRT_PREVIEW_REPO} == "True" ]]; then
+            # https://copr.fedorainfracloud.org/coprs/g/virtmaint-sig/virt-preview/
+            sudo dnf copr enable -y @virtmaint-sig/virt-preview
+        fi
+
        # Note that in CentOS/RHEL this needs to come from the RDO
        # repositories (qemu-kvm-ev ... which provides this package)
        # as the base system version is too old.  We should have
        # pre-installed these
        install_package qemu-kvm
+        install_package libvirt libvirt-devel python3-libvirt

-        install_package libvirt libvirt-devel
-        pip_uninstall libvirt-python
-        pip_install_gr libvirt-python
+        if is_arch "aarch64"; then
+            install_package edk2-aarch64
+        fi
    fi

    if [[ $DEBUG_LIBVIRT_COREDUMPS == True ]]; then
@@ -150,26 +160,24 @@ EOF
    fi

    if is_nova_console_proxy_compute_tls_enabled ; then
-        if is_service_enabled n-novnc ; then
-            echo "vnc_tls = 1" | sudo tee -a $QEMU_CONF
-            echo "vnc_tls_x509_verify = 1" | sudo tee -a $QEMU_CONF
+        echo "vnc_tls = 1" | sudo tee -a $QEMU_CONF
+        echo "vnc_tls_x509_verify = 1" | sudo tee -a $QEMU_CONF

-            sudo mkdir -p /etc/pki/libvirt-vnc
-            deploy_int_CA /etc/pki/libvirt-vnc/ca-cert.pem
-            deploy_int_cert /etc/pki/libvirt-vnc/server-cert.pem /etc/pki/libvirt-vnc/server-key.pem
-            # OpenSSL 1.1.0 generates the key file with permissions: 600, by
-            # default and the deploy_int* methods use 'sudo cp' to copy the
-            # files, making them owned by root:root.
-            # Change ownership of everything under /etc/pki/libvirt-vnc to
-            # libvirt-qemu:libvirt-qemu so that libvirt-qemu can read the key
-            # file.
-            sudo chown -R libvirt-qemu:libvirt-qemu /etc/pki/libvirt-vnc
-        fi
+        sudo mkdir -p /etc/pki/libvirt-vnc
+        deploy_int_CA /etc/pki/libvirt-vnc/ca-cert.pem
+        deploy_int_cert /etc/pki/libvirt-vnc/server-cert.pem /etc/pki/libvirt-vnc/server-key.pem
+        # OpenSSL 1.1.0 generates the key file with permissions: 600, by
+        # default and the deploy_int* methods use 'sudo cp' to copy the
+        # files, making them owned by root:root.
+        # Change ownership of everything under /etc/pki/libvirt-vnc to
+        # libvirt-qemu:libvirt-qemu so that libvirt-qemu can read the key
+        # file.
+        sudo chown -R libvirt-qemu:libvirt-qemu /etc/pki/libvirt-vnc
    fi

    # Service needs to be started on redhat/fedora -- do a restart for
    # sanity after fiddling the config.
-    restart_service $LIBVIRT_DAEMON
+    restart_service libvirtd

    # Restart virtlogd companion service to ensure it is running properly
    #  https://bugs.launchpad.net/ubuntu/+source/libvirt/+bug/1577455
@@ -39,19 +39,21 @@ function configure_nova_hypervisor {
    if ! is_ironic_hardware; then
        configure_libvirt
    fi
-    LIBVIRT_FIREWALL_DRIVER=${LIBVIRT_FIREWALL_DRIVER:-"nova.virt.firewall.NoopFirewallDriver"}

    iniset $NOVA_CONF DEFAULT compute_driver ironic.IronicDriver
-    iniset $NOVA_CONF DEFAULT firewall_driver $LIBVIRT_FIREWALL_DRIVER

    # ironic section
    iniset $NOVA_CONF ironic auth_type password
    iniset $NOVA_CONF ironic username admin
    iniset $NOVA_CONF ironic password $ADMIN_PASSWORD
-    iniset $NOVA_CONF ironic auth_url $KEYSTONE_AUTH_URI
-    iniset $NOVA_CONF ironic project_domain_id default
+    iniset $NOVA_CONF ironic auth_url $KEYSTONE_SERVICE_URI
+    if is_ironic_enforce_scope; then
+        iniset $NOVA_CONF ironic system_scope all
+    else
+        iniset $NOVA_CONF ironic project_domain_id default
+        iniset $NOVA_CONF ironic project_name demo
+    fi
    iniset $NOVA_CONF ironic user_domain_id default
-    iniset $NOVA_CONF ironic project_name demo
    iniset $NOVA_CONF ironic region_name $REGION_NAME

    # These are used with crufty legacy ironicclient
@@ -70,13 +72,6 @@ function install_nova_hypervisor {
        return
    fi
    install_libvirt
-    if [[ "$IRONIC_VM_LOG_CONSOLE" == "True" ]] && is_ubuntu; then
-        # Ubuntu packaging+apparmor issue prevents libvirt from loading
-        # the ROM from /usr/share/misc.  Workaround by installing it directly
-        # to a directory that it can read from. (LP: #1393548)
-        sudo rm -rf /usr/share/qemu/sgabios.bin
-        sudo cp /usr/share/misc/sgabios.bin /usr/share/qemu/sgabios.bin
-    fi
 }

 # start_nova_hypervisor - Start any required external services
@@ -91,7 +86,6 @@ function stop_nova_hypervisor {
    :
 }

-
 # Restore xtrace
 $_XTRACE_HYP_IRONIC

@@ -39,14 +39,15 @@ function cleanup_nova_hypervisor {
 function configure_nova_hypervisor {
    configure_libvirt
    iniset $NOVA_CONF libvirt virt_type "$LIBVIRT_TYPE"
-    iniset $NOVA_CONF libvirt cpu_mode "none"
+    iniset $NOVA_CONF libvirt cpu_mode "$LIBVIRT_CPU_MODE"
+    if [ "$LIBVIRT_CPU_MODE" == "custom" ] ; then
+        iniset $NOVA_CONF libvirt cpu_model "$LIBVIRT_CPU_MODEL"
+    fi
    # Do not enable USB tablet input devices to avoid QEMU CPU overhead.
    iniset $NOVA_CONF DEFAULT pointer_model "ps2mouse"
    iniset $NOVA_CONF libvirt live_migration_uri "qemu+ssh://$STACK_USER@%s/system"
    iniset $NOVA_CONF DEFAULT default_ephemeral_format "ext4"
    iniset $NOVA_CONF DEFAULT compute_driver "libvirt.LibvirtDriver"
-    LIBVIRT_FIREWALL_DRIVER=${LIBVIRT_FIREWALL_DRIVER:-"nova.virt.libvirt.firewall.IptablesFirewallDriver"}
-    iniset $NOVA_CONF DEFAULT firewall_driver "$LIBVIRT_FIREWALL_DRIVER"
    # Power architecture currently does not support graphical consoles.
    if is_arch "ppc64"; then
        iniset $NOVA_CONF vnc enabled "false"
@@ -54,8 +55,6 @@ function configure_nova_hypervisor {

    # arm64-specific configuration
    if is_arch "aarch64"; then
-        # arm64 architecture currently does not support graphical consoles.
-        iniset $NOVA_CONF vnc enabled "false"
        iniset $NOVA_CONF libvirt cpu_mode "host-passthrough"
    fi

@@ -104,7 +103,7 @@ function install_nova_hypervisor {

    if [[ "$ENABLE_FILE_INJECTION" == "True" ]] ; then
        if is_ubuntu; then
-            install_package python-guestfs
+            install_package python3-guestfs
            # NOTE(andreaf) Ubuntu kernel can only be read by root, which breaks libguestfs:
            # https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725)
            INSTALLED_KERNELS="$(ls /boot/vmlinuz-*)"
@@ -119,7 +118,7 @@ function install_nova_hypervisor {
            # Workaround for missing dependencies in python-libguestfs
            install_package python-libguestfs guestfs-data augeas augeas-lenses
        elif is_fedora; then
-            install_package python-libguestfs
+            install_package python3-libguestfs
        fi
    fi
 }
@@ -38,8 +38,6 @@ function cleanup_nova_hypervisor {
 function configure_nova_hypervisor {
    iniset $NOVA_CONF DEFAULT compute_driver "openvz.OpenVzDriver"
    iniset $NOVA_CONF DEFAULT connection_type "openvz"
-    LIBVIRT_FIREWALL_DRIVER=${LIBVIRT_FIREWALL_DRIVER:-"nova.virt.libvirt.firewall.IptablesFirewallDriver"}
-    iniset $NOVA_CONF DEFAULT firewall_driver "$LIBVIRT_FIREWALL_DRIVER"
 }

 # install_nova_hypervisor() - Install external components
@@ -1,110 +0,0 @@
-#!/bin/bash
-#
-# lib/nova_plugins/hypervisor-xenserver
-# Configure the XenServer hypervisor
-
-# Enable with:
-# VIRT_DRIVER=xenserver
-
-# Dependencies:
-# ``functions`` file
-# ``nova`` configuration
-
-# install_nova_hypervisor - install any external requirements
-# configure_nova_hypervisor - make configuration changes, including those to other services
-# start_nova_hypervisor - start any external services
-# stop_nova_hypervisor - stop any external services
-# cleanup_nova_hypervisor - remove transient data and cache
-
-# Save trace setting
-_XTRACE_XENSERVER=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-VNCSERVER_PROXYCLIENT_ADDRESS=${VNCSERVER_PROXYCLIENT_ADDRESS=169.254.0.1}
-
-
-# Entry Points
-# ------------
-
-# clean_nova_hypervisor - Clean up an installation
-function cleanup_nova_hypervisor {
-    # This function intentionally left blank
-    :
-}
-
-# configure_nova_hypervisor - Set config files, create data dirs, etc
-function configure_nova_hypervisor {
-    if [ -z "$XENAPI_CONNECTION_URL" ]; then
-        die $LINENO "XENAPI_CONNECTION_URL is not specified"
-    fi
-
-    # Check os-xenapi plugin is enabled
-    local plugins="${DEVSTACK_PLUGINS}"
-    local plugin
-    local found=0
-    for plugin in ${plugins//,/ }; do
-        if [[ "$plugin" = "os-xenapi" ]]; then
-            found=1
-            break
-        fi
-    done
-    if [[ $found -ne 1 ]]; then
-        die $LINENO "os-xenapi plugin is not specified. Please enable this plugin in local.conf"
-    fi
-
-    iniset $NOVA_CONF DEFAULT compute_driver "xenapi.XenAPIDriver"
-    iniset $NOVA_CONF xenserver connection_url "$XENAPI_CONNECTION_URL"
-    iniset $NOVA_CONF xenserver connection_username "$XENAPI_USER"
-    iniset $NOVA_CONF xenserver connection_password "$XENAPI_PASSWORD"
-    iniset $NOVA_CONF DEFAULT flat_injected "False"
-    # Need to avoid crash due to new firewall support
-    XEN_FIREWALL_DRIVER=${XEN_FIREWALL_DRIVER:-"nova.virt.firewall.IptablesFirewallDriver"}
-    iniset $NOVA_CONF DEFAULT firewall_driver "$XEN_FIREWALL_DRIVER"
-
-    local dom0_ip
-    dom0_ip=$(echo "$XENAPI_CONNECTION_URL" | cut -d "/" -f 3-)
-
-    local ssh_dom0
-    ssh_dom0="sudo -u $DOMZERO_USER ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$dom0_ip"
-
-    # install console logrotate script
-    tar -czf - -C $NOVA_DIR/tools/xenserver/ rotate_xen_guest_logs.sh |
-        $ssh_dom0 'tar -xzf - -C /root/ && chmod +x /root/rotate_xen_guest_logs.sh && mkdir -p /var/log/xen/guest'
-
-    # Create a cron job that will rotate guest logs
-    $ssh_dom0 crontab - << CRONTAB
-* * * * * /root/rotate_xen_guest_logs.sh >/dev/null 2>&1
-CRONTAB
-
-}
-
-# install_nova_hypervisor() - Install external components
-function install_nova_hypervisor {
-    # xenapi functionality is now included in os-xenapi library which houses the plugin
-    # so this function intentionally left blank
-    :
-}
-
-# start_nova_hypervisor - Start any required external services
-function start_nova_hypervisor {
-    # This function intentionally left blank
-    :
-}
-
-# stop_nova_hypervisor - Stop any external services
-function stop_nova_hypervisor {
-    # This function intentionally left blank
-    :
-}
-
-
-# Restore xtrace
-$_XTRACE_XENSERVER
-
-# Local variables:
-# mode: shell-script
-# End:
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# support vsctl or native.
+# until bug #1929446 is resolved we override the os-vif default
+# and fall back to the legacy "vsctl" driver.
+OS_VIF_OVS_OVSDB_INTERFACE=${OS_VIF_OVS_OVSDB_INTERFACE:="vsctl"}
+
+function is_ml2_ovs {
+    if [[ "${Q_AGENT}" == "openvswitch" ]]; then
+        echo "True"
+    fi
+    echo "False"
+}
+
+# This should be true for any ml2/ovs job but should be set to false for
+# all other ovs based jobs e.g. ml2/ovn
+OS_VIF_OVS_ISOLATE_VIF=${OS_VIF_OVS_ISOLATE_VIF:=$(is_ml2_ovs)}
+OS_VIF_OVS_ISOLATE_VIF=$(trueorfalse False OS_VIF_OVS_ISOLATE_VIF)
+
+function configure_os_vif {
+    if [[ -e ${NOVA_CONF} ]]; then
+        iniset ${NOVA_CONF} os_vif_ovs ovsdb_interface ${OS_VIF_OVS_OVSDB_INTERFACE}
+        iniset ${NOVA_CONF} os_vif_ovs isolate_vif ${OS_VIF_OVS_ISOLATE_VIF}
+    fi
+    if [[ -e ${NEUTRON_CONF} ]]; then
+        iniset ${NEUTRON_CONF} os_vif_ovs ovsdb_interface ${OS_VIF_OVS_OVSDB_INTERFACE}
+        iniset ${NEUTRON_CONF} os_vif_ovs isolate_vif ${OS_VIF_OVS_ISOLATE_VIF}
+    fi
+}
@@ -144,11 +144,10 @@ function install_placement {
 # start_placement_api() - Start the API processes ahead of other things
 function start_placement_api {
    if [[ "$WSGI_MODE" == "uwsgi" ]]; then
-        run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF"
+        run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF"
    else
        enable_apache_site placement-api
        restart_apache_server
-        tail_log placement-api /var/log/$APACHE_NAME/placement-api.log
    fi

    echo "Waiting for placement-api to start..."
@@ -66,7 +66,12 @@ EOF
            sudo systemctl restart epmd.socket epmd.service
        fi
        if is_fedora || is_suse; then
-            sudo systemctl enable rabbitmq-server
+            # NOTE(jangutter): If rabbitmq is not running (as in a fresh
+            # install) then rabbit_setuser triggers epmd@0.0.0.0.socket with
+            # socket activation. This fails the first time and does not get
+            # cleared. It is benign, but the workaround is to start rabbitmq a
+            # bit earlier for RPM based distros.
+            sudo systemctl --now enable rabbitmq-server
        fi
    fi
 }
@@ -179,12 +179,9 @@ function is_swift_enabled {
 # cleanup_swift() - Remove residual data files
 function cleanup_swift {
    rm -f ${SWIFT_CONF_DIR}{*.builder,*.ring.gz,backups/*.builder,backups/*.ring.gz}
-    if egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then
-        sudo umount ${SWIFT_DATA_DIR}/drives/sdb1
-    fi
-    if [[ -e ${SWIFT_DISK_IMAGE} ]]; then
-        rm ${SWIFT_DISK_IMAGE}
-    fi
+
+    destroy_disk ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1
+
    rm -rf ${SWIFT_DATA_DIR}/run/
    if [ "$SWIFT_USE_MOD_WSGI" == "True" ]; then
        _cleanup_swift_apache_wsgi
@@ -335,7 +332,6 @@ function configure_swift {
    local node_number
    local swift_node_config
    local swift_log_dir
-    local user_group

    # Make sure to kill all swift processes first
    $SWIFT_BIN_DIR/swift-init --run-dir=${SWIFT_DATA_DIR}/run all stop || true
@@ -353,7 +349,7 @@ function configure_swift {
    # partitions (which make more sense when you have a multi-node
    # setup) we configure it with our version of rsync.
    sed -e "
-        s/%GROUP%/${USER_GROUP}/;
+        s/%GROUP%/$(id -g -n ${STACK_USER})/;
        s/%USER%/${STACK_USER}/;
        s,%SWIFT_DATA_DIR%,$SWIFT_DATA_DIR,;
    " $FILES/swift/rsyncd.conf | sudo tee /etc/rsyncd.conf
@@ -406,6 +402,11 @@ function configure_swift {
    # Versioned Writes
    iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:versioned_writes allow_versioned_writes true

+    # Add sha1 temporary https://storyboard.openstack.org/#!/story/2010068
+    if [[ "$SWIFT_ENABLE_TEMPURLS" == "True" ]]; then
+        iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:tempurl allowed_digests "sha1 sha256 sha512"
+    fi
+
    # Configure Ceilometer
    if is_service_enabled ceilometer; then
        iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:ceilometer "set log_level" "WARN"
@@ -428,10 +429,13 @@ function configure_swift {
        swift_pipeline+=" s3api"
    fi
    if is_service_enabled keystone; then
+        swift_pipeline+=" authtoken"
        if is_service_enabled s3api;then
            swift_pipeline+=" s3token"
+            iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token auth_uri ${KEYSTONE_SERVICE_URI_V3}
+            iniset ${SWIFT_CONFIG_PROXY_SERVER} filter:s3token delay_auth_decision true
        fi
-        swift_pipeline+=" authtoken keystoneauth"
+        swift_pipeline+=" keystoneauth"
    fi

    swift_pipeline+=" tempauth "
@@ -519,12 +523,12 @@ function configure_swift {
        local auth_vers
        auth_vers=$(iniget ${testfile} func_test auth_version)
        iniset ${testfile} func_test auth_host ${KEYSTONE_SERVICE_HOST}
-        if [[ "$KEYSTONE_AUTH_PROTOCOL" == "https" ]]; then
+        if [[ "$KEYSTONE_SERVICE_PROTOCOL" == "https" ]]; then
            iniset ${testfile} func_test auth_port 443
        else
            iniset ${testfile} func_test auth_port 80
        fi
-        iniset ${testfile} func_test auth_uri ${KEYSTONE_AUTH_URI}
+        iniset ${testfile} func_test auth_uri ${KEYSTONE_SERVICE_URI}
        if [[ "$auth_vers" == "3" ]]; then
            iniset ${testfile} func_test auth_prefix /identity/v3/
        else
@@ -573,28 +577,7 @@ function create_swift_disk {
    sudo install -d -o ${STACK_USER} -g ${user_group} ${SWIFT_DATA_DIR}/{drives,cache,run,logs}

    # Create a loopback disk and format it to XFS.
-    if [[ -e ${SWIFT_DISK_IMAGE} ]]; then
-        if egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then
-            sudo umount ${SWIFT_DATA_DIR}/drives/sdb1
-            sudo rm -f ${SWIFT_DISK_IMAGE}
-        fi
-    fi
-
-    mkdir -p ${SWIFT_DATA_DIR}/drives/images
-    sudo touch ${SWIFT_DISK_IMAGE}
-    sudo chown ${STACK_USER}: ${SWIFT_DISK_IMAGE}
-
-    truncate -s ${SWIFT_LOOPBACK_DISK_SIZE} ${SWIFT_DISK_IMAGE}
-
-    # Make a fresh XFS filesystem
-    /sbin/mkfs.xfs -f -i size=1024  ${SWIFT_DISK_IMAGE}
-
-    # Mount the disk with mount options to make it as efficient as possible
-    mkdir -p ${SWIFT_DATA_DIR}/drives/sdb1
-    if ! egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then
-        sudo mount -t xfs -o loop,noatime,nodiratime,logbufs=8  \
-            ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1
-    fi
+    create_disk ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1 ${SWIFT_LOOPBACK_DISK_SIZE}

    # Create a link to the above mount and
    # create all of the directories needed to emulate a few different servers
@@ -738,7 +721,9 @@ function init_swift {

 function install_swift {
    git_clone $SWIFT_REPO $SWIFT_DIR $SWIFT_BRANCH
-    setup_develop $SWIFT_DIR
+    # keystonemiddleware needs to be installed via keystone extras as defined
+    # in setup.cfg, see bug #1909018 for more details.
+    setup_develop $SWIFT_DIR keystone
    if [ "$SWIFT_USE_MOD_WSGI" == "True" ]; then
        install_apache_wsgi
    fi
@@ -862,12 +847,15 @@ function stop_swift {

 function swift_configure_tempurls {
    # note we are using swift credentials!
-    OS_USERNAME=swift \
-    OS_PASSWORD=$SERVICE_PASSWORD \
-    OS_USER_DOMAIN_NAME=$SERVICE_DOMAIN_NAME \
-    OS_PROJECT_NAME=$SERVICE_PROJECT_NAME \
-    OS_PROJECT_DOMAIN_NAME=$SERVICE_DOMAIN_NAME \
-    openstack object store account \
+    openstack --os-cloud "" \
+        --os-region-name $REGION_NAME \
+        --os-auth-url $KEYSTONE_SERVICE_URI \
+        --os-username=swift \
+        --os-password=$SERVICE_PASSWORD \
+        --os-user-domain-name=$SERVICE_DOMAIN_NAME \
+        --os-project-name=$SERVICE_PROJECT_NAME \
+        --os-project-domain-name=$SERVICE_DOMAIN_NAME \
+        object store account \
        set --property "Temp-URL-Key=$SWIFT_TEMPURL_KEY"
 }

--- a/Show More
+++ b/Show More