Merge "Replace deprecated datetime.utcnow()"

Merge "Globally skip devstack job for pre-commit config update"
Merge "Add image format enforcement toggle"
2024-10-25 08:38:40 +00:00 · 2024-10-25 07:56:22 +00:00 · 2024-10-25 07:56:19 +00:00 · 2024-10-24 00:54:28 +09:00 · 2024-10-21 14:22:36 +00:00 · 2024-10-19 12:55:43 +00:00
46 changed files with 1099 additions and 531 deletions
@@ -38,3 +38,5 @@ stack-screenrc
 userrc_early
 AUTHORS
 ChangeLog
+tools/dbcounter/build/
+tools/dbcounter/dbcounter.egg-info/
@@ -1,18 +1,18 @@
 - nodeset:
-    name: openstack-single-node
+    name: openstack-single-node-jammy
    nodes:
      - name: controller
-        label: ubuntu-xenial
+        label: ubuntu-jammy
    groups:
      - name: tempest
        nodes:
          - controller

 - nodeset:
-    name: openstack-single-node-jammy
+    name: openstack-single-node-noble
    nodes:
      - name: controller
-        label: ubuntu-jammy
+        label: ubuntu-noble
    groups:
      - name: tempest
        nodes:
@@ -38,26 +38,6 @@
        nodes:
          - controller

- nodeset:
-    name: openstack-single-node-xenial
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: devstack-single-node-centos-7
-    nodes:
-      - name: controller
-        label: centos-7
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
 - nodeset:
    name: devstack-single-node-centos-9-stream
    nodes:
@@ -88,6 +68,9 @@
        nodes:
          - controller

+# Note(sean-k-mooney): this is still used by horizon for
+# horizon-integration-tests, horizon-integration-pytest and
+# horizon-ui-pytest, remove when horizon is updated.
 - nodeset:
    name: devstack-single-node-debian-bullseye
    nodes:
@@ -108,46 +91,6 @@
        nodes:
          - controller

- nodeset:
-    name: devstack-single-node-openeuler-22.03
-    nodes:
-      - name: controller
-        label: openEuler-22-03-LTS
-    groups:
-      - name: tempest
-        nodes:
-          - controller
-
- nodeset:
-    name: openstack-two-node
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-      - name: compute1
-        label: ubuntu-xenial
-    groups:
-      # Node where tests are executed and test results collected
-      - name: tempest
-        nodes:
-          - controller
-      # Nodes running the compute service
-      - name: compute
-        nodes:
-          - controller
-          - compute1
-      # Nodes that are not the controller
-      - name: subnode
-        nodes:
-          - compute1
-      # Switch node for multinode networking setup
-      - name: switch
-        nodes:
-          - controller
-      # Peer nodes for multinode networking setup
-      - name: peers
-        nodes:
-          - compute1
-
 - nodeset:
    name: openstack-two-node-centos-9-stream
    nodes:
@@ -208,6 +151,36 @@
        nodes:
          - compute1

+- nodeset:
+    name: openstack-two-node-noble
+    nodes:
+      - name: controller
+        label: ubuntu-noble
+      - name: compute1
+        label: ubuntu-noble
+    groups:
+      # Node where tests are executed and test results collected
+      - name: tempest
+        nodes:
+          - controller
+      # Nodes running the compute service
+      - name: compute
+        nodes:
+          - controller
+          - compute1
+      # Nodes that are not the controller
+      - name: subnode
+        nodes:
+          - compute1
+      # Switch node for multinode networking setup
+      - name: switch
+        nodes:
+          - controller
+      # Peer nodes for multinode networking setup
+      - name: peers
+        nodes:
+          - compute1
+
 - nodeset:
    name: openstack-two-node-focal
    nodes:
@@ -268,36 +241,6 @@
        nodes:
          - compute1

- nodeset:
-    name: openstack-two-node-xenial
-    nodes:
-      - name: controller
-        label: ubuntu-xenial
-      - name: compute1
-        label: ubuntu-xenial
-    groups:
-      # Node where tests are executed and test results collected
-      - name: tempest
-        nodes:
-          - controller
-      # Nodes running the compute service
-      - name: compute
-        nodes:
-          - controller
-          - compute1
-      # Nodes that are not the controller
-      - name: subnode
-        nodes:
-          - compute1
-      # Switch node for multinode networking setup
-      - name: switch
-        nodes:
-          - controller
-      # Peer nodes for multinode networking setup
-      - name: peers
-        nodes:
-          - compute1
-
 - nodeset:
    name: openstack-three-node-focal
    nodes:
@@ -489,6 +432,8 @@
      - ^releasenotes/.*$
      # Translations
      - ^.*/locale/.*po$
+      # pre-commit config
+      - ^.pre-commit-config.yaml$

 - job:
    name: devstack-minimal
@@ -513,6 +458,7 @@
        file_tracker: true
        mysql: true
        rabbit: true
+        openstack-cli-server: true
    group-vars:
      subnode:
        devstack_services:
@@ -520,6 +466,7 @@
          dstat: false
          memory_tracker: true
          file_tracker: true
+          openstack-cli-server: true
        devstack_localrc:
          # Multinode specific settings
          HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}"
@@ -565,9 +512,17 @@
      - opendev.org/openstack/nova
      - opendev.org/openstack/placement
      - opendev.org/openstack/swift
+      - opendev.org/openstack/os-test-images
    timeout: 7200
    vars:
-      configure_swap_size: 4096
+      # based on observation of the integrated gate
+      # tempest-integrated-compute was only using ~1.7GB of swap
+      # when zswap and the host turning are enabled that increase
+      # slightly to ~2GB. we are setting the swap size to 8GB to
+      # be safe and account for more complex scenarios.
+      # we should revisit this value after some time to see if we
+      # can reduce it.
+      configure_swap_size: 8192
      devstack_localrc:
        # Common OpenStack services settings
        SWIFT_REPLICAS: 1
@@ -576,6 +531,26 @@
        DEBUG_LIBVIRT_COREDUMPS: true
        NOVA_VNC_ENABLED: true
        OVN_DBS_LOG_LEVEL: dbg
+        # tune the host to optimize memory usage and hide io latency
+        # these setting will configure the kernel to treat the host page
+        # cache and swap with equal priority, and prefer deferring writes
+        # changing the default swappiness, dirty_ratio and
+        # the vfs_cache_pressure
+        ENABLE_SYSCTL_MEM_TUNING: true
+        # the net tuning optimizes ipv4 tcp fast open and config the default
+        # qdisk policy to pfifo_fast which effectively disable all qos.
+        # this minimizes the cpu load of the host network stack
+        ENABLE_SYSCTL_NET_TUNING: true
+        # zswap allows the kernel to compress pages in memory before swapping
+        # them to disk. this can reduce the amount of swap used and improve
+        # performance. effectively this trades a small amount of cpu for an
+        # increase in swap performance by reducing the amount of data
+        # written to disk. the overall speedup is proportional to the
+        # compression ratio and the speed of the swap device.
+        # NOTE: this option is ignored when not using nova with the libvirt
+        # virt driver.
+        NOVA_LIBVIRT_TB_CACHE_SIZE: 128
+        ENABLE_ZSWAP: true
      devstack_local_conf:
        post-config:
          $NEUTRON_CONF:
@@ -668,6 +643,26 @@
          Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}"
          NOVA_VNC_ENABLED: true
          ENABLE_CHASSIS_AS_GW: false
+          # tune the host to optimize memory usage and hide io latency
+          # these setting will configure the kernel to treat the host page
+          # cache and swap with equal priority, and prefer deferring writes
+          # changing the default swappiness, dirty_ratio and
+          # the vfs_cache_pressure
+          ENABLE_SYSCTL_MEM_TUNING: true
+          # the net tuning optimizes ipv4 tcp fast open and config the default
+          # qdisk policy to pfifo_fast which effectively disable all qos.
+          # this minimizes the cpu load of the host network stack
+          ENABLE_SYSCTL_NET_TUNING: true
+          # zswap allows the kernel to compress pages in memory before swapping
+          # them to disk. this can reduce the amount of swap used and improve
+          # performance. effectivly this trades a small amount of cpu for an
+          # increase in swap performance by reducing the amount of data
+          # written to disk. the overall speedup is porportional to the
+          # compression ratio and the speed of the swap device.
+          ENABLE_ZSWAP: true
+          # NOTE: this option is ignored when not using nova with the libvirt
+          # virt driver.
+          NOVA_LIBVIRT_TB_CACHE_SIZE: 128

 - job:
    name: devstack-ipv6
@@ -716,24 +711,8 @@
    description: Debian Bookworm platform test
    nodeset: devstack-single-node-debian-bookworm
    timeout: 9000
-    voting: false
    vars:
      configure_swap_size: 4096
-      devstack_localrc:
-        # TODO(frickler): drop this once wheel build is fixed
-        MYSQL_GATHER_PERFORMANCE: false
-
- job:
-    name: devstack-platform-debian-bullseye
-    parent: tempest-full-py3
-    description: Debian Bullseye platform test
-    nodeset: devstack-single-node-debian-bullseye
-    timeout: 9000
-    vars:
-      configure_swap_size: 4096
-      devstack_localrc:
-        # TODO(frickler): drop this once wheel build is fixed
-        MYSQL_GATHER_PERFORMANCE: false

 - job:
    name: devstack-platform-rocky-blue-onyx
@@ -741,9 +720,23 @@
    description: Rocky Linux 9 Blue Onyx platform test
    nodeset: devstack-single-node-rockylinux-9
    timeout: 9000
+    # NOTE(danms): This has been failing lately with some repository metadata
+    # errors. We're marking this as non-voting until it appears to have
+    # stabilized:
+    # https://zuul.openstack.org/builds?job_name=devstack-platform-rocky-blue-onyx&skip=0
+    voting: false
    vars:
      configure_swap_size: 4096

+- job:
+    name: devstack-platform-ubuntu-noble
+    parent: tempest-full-py3
+    description: Ubuntu 24.04 LTS (noble) platform test
+    nodeset: openstack-single-node-noble
+    timeout: 9000
+    vars:
+      configure_swap_size: 8192
+
 - job:
    name: devstack-platform-ubuntu-jammy-ovn-source
    parent: devstack-platform-ubuntu-jammy
@@ -795,62 +788,6 @@
          # Enable Neutron ML2/OVS services
          q-agt: true

- job:
-    name: devstack-platform-openEuler-22.03-ovn-source
-    parent: tempest-full-py3
-    description: openEuler 22.03 LTS platform test (OVN)
-    nodeset: devstack-single-node-openeuler-22.03
-    voting: false
-    timeout: 9000
-    vars:
-      configure_swap_size: 4096
-      devstack_localrc:
-        # NOTE(wxy):  OVN package is not supported by openEuler yet. Build it
-        # from source instead.
-        OVN_BUILD_FROM_SOURCE: True
-        OVN_BRANCH: "v21.06.0"
-        OVS_BRANCH: "a4b04276ab5934d087669ff2d191a23931335c87"
-        OVS_SYSCONFDIR: "/usr/local/etc/openvswitch"
-
- job:
-    name: devstack-platform-openEuler-22.03-ovs
-    parent: tempest-full-py3
-    description: openEuler 22.03 LTS platform test (OVS)
-    nodeset: devstack-single-node-openeuler-22.03
-    voting: false
-    timeout: 9000
-    vars:
-      configure_swap_size: 8192
-      devstack_localrc:
-        Q_AGENT: openvswitch
-        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
-        Q_ML2_TENANT_NETWORK_TYPE: vxlan
-      devstack_services:
-        # Disable OVN services
-        ovn-northd: false
-        ovn-controller: false
-        ovs-vswitchd: false
-        ovsdb-server: false
-        # Disable Neutron ML2/OVN services
-        q-ovn-metadata-agent: false
-        # Enable Neutron ML2/OVS services
-        q-agt: true
-        q-dhcp: true
-        q-l3: true
-        q-meta: true
-        q-metering: true
-    group-vars:
-      subnode:
-        devstack_services:
-          # Disable OVN services
-          ovn-controller: false
-          ovs-vswitchd: false
-          ovsdb-server: false
-          # Disable Neutron ML2/OVN services
-          q-ovn-metadata-agent: false
-          # Enable Neutron ML2/OVS services
-          q-agt: true
-
 - job:
    name: devstack-no-tls-proxy
    parent: tempest-full-py3
@@ -937,10 +874,10 @@
        - devstack-enforce-scope
        - devstack-platform-centos-9-stream
        - devstack-platform-debian-bookworm
-        - devstack-platform-debian-bullseye
        - devstack-platform-rocky-blue-onyx
        - devstack-platform-ubuntu-jammy-ovn-source
        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
        - devstack-multinode
        - devstack-unit-tests
        - openstack-tox-bashate
@@ -954,10 +891,6 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-linuxbridge-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - neutron-ovn-tempest-ovs-release:
            voting: false
            irrelevant-files:
@@ -984,8 +917,11 @@
      jobs:
        - devstack
        - devstack-ipv6
-        - devstack-platform-debian-bullseye
-        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-debian-bookworm
+        - devstack-platform-ubuntu-noble
+        # NOTE(danms): Disabled due to instability, see comment in the job
+        # definition above.
+        # - devstack-platform-rocky-blue-onyx
        - devstack-enforce-scope
        - devstack-multinode
        - devstack-unit-tests
@@ -994,10 +930,6 @@
            irrelevant-files:
              - ^.*\.rst$
              - ^doc/.*$
-        - neutron-linuxbridge-tempest:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
        - ironic-tempest-bios-ipmi-direct-tinyipa
        - swift-dsvm-functional
        - grenade:
@@ -1021,7 +953,9 @@
    # pruned.
    #
    # * nova-next: maintained by nova for unreleased/undefaulted
-    #    things
+    #    things, this job is not experimental but often is used to test
+    #    things that are not yet production ready or to test what will be
+    #    the new default after a deprecation period has ended.
    # * neutron-fullstack-with-uwsgi: maintained by neutron for fullstack test
    #    when neutron-api is served by uwsgi, it's in exprimental for testing.
    #    the next cycle we can remove this  job if things turn out to be
@@ -1031,7 +965,7 @@
    #    stable engouh with uwsgi.
    # * neutron-ovn-tempest-with-uwsgi: maintained by neutron for tempest test.
    #    Next cycle we can remove this if everything run out stable enough.
-    # * nova-multi-cell: maintained by nova and currently non-voting in the
+    # * nova-multi-cell: maintained by nova and now is voting in the
    #    check queue for nova changes but relies on devstack configuration

    experimental:
@@ -1065,3 +999,11 @@
    periodic:
      jobs:
        - devstack-no-tls-proxy
+    periodic-weekly:
+      jobs:
+        - devstack-platform-centos-9-stream
+        - devstack-platform-debian-bookworm
+        - devstack-platform-rocky-blue-onyx
+        - devstack-platform-ubuntu-jammy-ovn-source
+        - devstack-platform-ubuntu-jammy-ovs
+        - devstack-platform-ubuntu-noble
@@ -4,8 +4,4 @@ Pygments
 docutils
 sphinx>=2.0.0,!=2.1.0 # BSD
 openstackdocstheme>=2.2.1 # Apache-2.0
-nwdiag
-blockdiag
-sphinxcontrib-blockdiag
-sphinxcontrib-nwdiag
 zuul-sphinx>=0.2.0
@@ -23,14 +23,14 @@

 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinx.ext.autodoc',
-               'zuul_sphinx',
-               'openstackdocstheme',
-               'sphinxcontrib.blockdiag',
-               'sphinxcontrib.nwdiag' ]
+extensions = [
+    'sphinx.ext.autodoc',
+    'zuul_sphinx',
+    'openstackdocstheme',
+]

 # openstackdocstheme options
-openstackdocs_repo_name = 'openstack-dev/devstack'
+openstackdocs_repo_name = 'openstack/devstack'
 openstackdocs_pdf_link = True
 openstackdocs_bug_project = 'devstack'
 openstackdocs_bug_tag = ''
@@ -41,19 +41,8 @@ network and is on a shared subnet with other machines.  The
 `local.conf` exhibited here assumes that 1500 is a reasonable MTU to
 use on that network.

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                }
-        }
+.. image:: /assets/images/neutron-network-1.png
+   :alt: Network configuration for a single DevStack node


 DevStack Configuration
@@ -100,21 +89,8 @@ also want to do multinode testing and networking.
 Physical Network Setup
 ~~~~~~~~~~~~~~~~~~~~~~

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                        devstack-2 [ address = "172.18.161.7" ];
-                }
-        }
-
+.. image:: /assets/images/neutron-network-2.png
+   :alt: Network configuration for multiple DevStack nodes

 After DevStack installs and configures Neutron, traffic from guest VMs
 flows out of `devstack-2` (the compute node) and is encapsulated in a
@@ -222,8 +198,6 @@ connect OpenStack nodes (like `devstack-2`) together. This bridge is
 used so that project network traffic, using the VXLAN tunneling
 protocol, flows between each compute node where project instances run.

-
-
 DevStack Compute Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -268,30 +242,8 @@ to the neutron L3 service.
 Physical Network Setup
 ----------------------

-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network provider_net {
-                        address = "203.0.113.0/24"
-                        router [ address = "203.0.113.1" ];
-                        controller;
-                        compute1;
-                        compute2;
-                }
-
-                network control_plane {
-                        router [ address = "10.0.0.1" ]
-                        address = "10.0.0.0/24"
-                        controller [ address = "10.0.0.2" ]
-                        compute1 [ address = "10.0.0.3" ]
-                        compute2 [ address = "10.0.0.4" ]
-                }
-        }
-
+.. image:: /assets/images/neutron-network-3.png
+   :alt: Network configuration for provider networks

 On a compute node, the first interface, eth0 is used for the OpenStack
 management (API, message bus, etc) as well as for ssh for an
@@ -122,7 +122,7 @@ when creating the server, for example:
 .. code-block:: shell

  $ openstack --os-compute-api-version 2.37 server create --flavor cirros256 \
-      --image cirros-0.6.2-x86_64-disk --nic none --wait test-server
+      --image cirros-0.6.3-x86_64-disk --nic none --wait test-server

 .. note:: ``--os-compute-api-version`` greater than or equal to 2.37 is
          required to use ``--nic=none``.
@@ -133,6 +133,8 @@ there.
 You can ``source openrc`` in your shell, and then use the
 ``openstack`` command line tool to manage your devstack.

+You can :ref:`create a VM and SSH into it <ssh>`.
+
 You can ``cd /opt/stack/tempest`` and run tempest tests that have
 been configured to work with your devstack.

@@ -68,7 +68,7 @@ Shared Guest Interface
 .. warning::

   This is not a recommended configuration. Because of interactions
-   between ovs and bridging, if you reboot your box with active
+   between OVS and bridging, if you reboot your box with active
   networking you may lose network connectivity to your system.

 If you need your guests accessible on the network, but only have 1
@@ -114,3 +114,125 @@ For IPv6, ``FIXED_RANGE_V6`` will default to the first /64 of the value of
 ``FIXED_RANGE_V6`` will just use the value of that directly.
 ``SUBNETPOOL_PREFIX_V6`` will just default to the value of
 ``IPV6_ADDRS_SAFE_TO_USE`` directly.
+
+.. _ssh:
+
+SSH access to instances
+=======================
+
+To validate connectivity, you can create an instance using the
+``$PRIVATE_NETWORK_NAME`` network (default: ``private``), create a floating IP
+using the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``), and attach
+this floating IP to the instance:
+
+.. code-block:: shell
+
+    openstack keypair create --public-key ~/.ssh/id_rsa.pub test-keypair
+    openstack server create --network private --key-name test-keypair ... test-server
+    fip_id=$(openstack floating ip create public -f value -c id)
+    openstack server add floating ip test-server ${fip_id}
+
+Once done, ensure you have enabled SSH and ICMP (ping) access for the security
+group used for the instance. You can either create a custom security group and
+specify it when creating the instance or add it after creation, or you can
+modify the ``default`` security group created by default for each project.
+Let's do the latter:
+
+.. code-block:: shell
+
+    openstack security group rule create --proto icmp --dst-port 0 default
+    openstack security group rule create --proto tcp --dst-port 22 default
+
+Finally, SSH into the instance. If you used the Cirros instance uploaded by
+default, then you can run the following:
+
+.. code-block:: shell
+
+    openstack server ssh test-server -- -l cirros
+
+This will connect using the ``cirros`` user and the keypair you configured when
+creating the instance.
+
+Remote SSH access to instances
+==============================
+
+You can also SSH to created instances on your DevStack host from other hosts.
+This can be helpful if you are e.g. deploying DevStack in a VM on an existing
+cloud and wish to do development on your local machine. There are a few ways to
+do this.
+
+.. rubric:: Configure instances to be locally accessible
+
+The most obvious way is to configure guests to be locally accessible, as
+described `above <Locally Accessible Guests>`__. This has the advantage of
+requiring no further effort on the client. However, it is more involved and
+requires either support from your cloud or some inadvisable workarounds.
+
+.. rubric:: Use your DevStack host as a jump host
+
+You can choose to use your DevStack host as a jump host. To SSH to a instance
+this way, pass the standard ``-J`` option to the ``openstack ssh`` / ``ssh``
+command. For example:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -J username@devstack-host
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+This can also be configured via your ``~/.ssh/config`` file, making it rather
+effortless. However, it only allows SSH access. If you want to access e.g. a
+web application on the instance, you will need to configure an SSH tunnel and
+forward select ports using the ``-L`` option. For example, to forward HTTP
+traffic:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -L 8080:username@devstack-host:80
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+As you can imagine, this can quickly get out of hand, particularly for more
+complex guest applications with multiple ports.
+
+.. rubric:: Use a proxy or VPN tool
+
+You can use a proxy or VPN tool to enable tunneling for the floating IP
+address range of the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``)
+defined by ``$FLOATING_RANGE`` (default: ``172.24.4.0/24``). There are many
+such tools available to do this. For example, we could use a useful utility
+called `shuttle`__. To enable tunneling using ``shuttle``, first ensure you
+have allowed SSH and HTTP(S) traffic to your DevStack host. Allowing HTTP(S)
+traffic is necessary so you can use the OpenStack APIs remotely. How you do
+this will depend on where your DevStack host is running. Once this is done,
+install ``sshuttle`` on your localhost:
+
+.. code-block:: bash
+
+    sudo apt-get install sshuttle || yum install sshuttle
+
+Finally, start ``sshuttle`` on your localhost using the floating IP address
+range. For example, assuming you are using the default value for
+``$FLOATING_RANGE``, you can do:
+
+.. code-block:: bash
+
+    sshuttle -r username@devstack-host 172.24.4.0/24
+
+(where ``username`` and ``devstack-host`` are the username and hostname of your
+DevStack host).
+
+You should now be able to create an instance and SSH into it:
+
+.. code-block:: bash
+
+    openstack server ssh test-server -- -l cirros
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`)
+
+.. __: https://github.com/sshuttle/sshuttle
@@ -28,8 +28,6 @@ openstack/aodh                           `https://opendev.org/openstack/aodh <ht
 openstack/barbican                       `https://opendev.org/openstack/barbican <https://opendev.org/openstack/barbican>`__
 openstack/blazar                         `https://opendev.org/openstack/blazar <https://opendev.org/openstack/blazar>`__
 openstack/ceilometer                     `https://opendev.org/openstack/ceilometer <https://opendev.org/openstack/ceilometer>`__
-openstack/ceilometer-powervm             `https://opendev.org/openstack/ceilometer-powervm <https://opendev.org/openstack/ceilometer-powervm>`__
-openstack/cinderlib                      `https://opendev.org/openstack/cinderlib <https://opendev.org/openstack/cinderlib>`__
 openstack/cloudkitty                     `https://opendev.org/openstack/cloudkitty <https://opendev.org/openstack/cloudkitty>`__
 openstack/cyborg                         `https://opendev.org/openstack/cyborg <https://opendev.org/openstack/cyborg>`__
 openstack/designate                      `https://opendev.org/openstack/designate <https://opendev.org/openstack/designate>`__
@@ -39,7 +37,6 @@ openstack/devstack-plugin-container      `https://opendev.org/openstack/devstack
 openstack/devstack-plugin-kafka          `https://opendev.org/openstack/devstack-plugin-kafka <https://opendev.org/openstack/devstack-plugin-kafka>`__
 openstack/devstack-plugin-nfs            `https://opendev.org/openstack/devstack-plugin-nfs <https://opendev.org/openstack/devstack-plugin-nfs>`__
 openstack/devstack-plugin-open-cas       `https://opendev.org/openstack/devstack-plugin-open-cas <https://opendev.org/openstack/devstack-plugin-open-cas>`__
-openstack/ec2-api                        `https://opendev.org/openstack/ec2-api <https://opendev.org/openstack/ec2-api>`__
 openstack/freezer                        `https://opendev.org/openstack/freezer <https://opendev.org/openstack/freezer>`__
 openstack/freezer-api                    `https://opendev.org/openstack/freezer-api <https://opendev.org/openstack/freezer-api>`__
 openstack/freezer-tempest-plugin         `https://opendev.org/openstack/freezer-tempest-plugin <https://opendev.org/openstack/freezer-tempest-plugin>`__
@@ -51,9 +48,7 @@ openstack/ironic-inspector               `https://opendev.org/openstack/ironic-i
 openstack/ironic-prometheus-exporter     `https://opendev.org/openstack/ironic-prometheus-exporter <https://opendev.org/openstack/ironic-prometheus-exporter>`__
 openstack/ironic-ui                      `https://opendev.org/openstack/ironic-ui <https://opendev.org/openstack/ironic-ui>`__
 openstack/keystone                       `https://opendev.org/openstack/keystone <https://opendev.org/openstack/keystone>`__
-openstack/kuryr-kubernetes               `https://opendev.org/openstack/kuryr-kubernetes <https://opendev.org/openstack/kuryr-kubernetes>`__
 openstack/kuryr-libnetwork               `https://opendev.org/openstack/kuryr-libnetwork <https://opendev.org/openstack/kuryr-libnetwork>`__
-openstack/kuryr-tempest-plugin           `https://opendev.org/openstack/kuryr-tempest-plugin <https://opendev.org/openstack/kuryr-tempest-plugin>`__
 openstack/magnum                         `https://opendev.org/openstack/magnum <https://opendev.org/openstack/magnum>`__
 openstack/magnum-ui                      `https://opendev.org/openstack/magnum-ui <https://opendev.org/openstack/magnum-ui>`__
 openstack/manila                         `https://opendev.org/openstack/manila <https://opendev.org/openstack/manila>`__
@@ -64,12 +59,10 @@ openstack/mistral                        `https://opendev.org/openstack/mistral
 openstack/monasca-api                    `https://opendev.org/openstack/monasca-api <https://opendev.org/openstack/monasca-api>`__
 openstack/monasca-events-api             `https://opendev.org/openstack/monasca-events-api <https://opendev.org/openstack/monasca-events-api>`__
 openstack/monasca-tempest-plugin         `https://opendev.org/openstack/monasca-tempest-plugin <https://opendev.org/openstack/monasca-tempest-plugin>`__
-openstack/murano                         `https://opendev.org/openstack/murano <https://opendev.org/openstack/murano>`__
 openstack/networking-bagpipe             `https://opendev.org/openstack/networking-bagpipe <https://opendev.org/openstack/networking-bagpipe>`__
 openstack/networking-baremetal           `https://opendev.org/openstack/networking-baremetal <https://opendev.org/openstack/networking-baremetal>`__
 openstack/networking-bgpvpn              `https://opendev.org/openstack/networking-bgpvpn <https://opendev.org/openstack/networking-bgpvpn>`__
 openstack/networking-generic-switch      `https://opendev.org/openstack/networking-generic-switch <https://opendev.org/openstack/networking-generic-switch>`__
-openstack/networking-powervm             `https://opendev.org/openstack/networking-powervm <https://opendev.org/openstack/networking-powervm>`__
 openstack/networking-sfc                 `https://opendev.org/openstack/networking-sfc <https://opendev.org/openstack/networking-sfc>`__
 openstack/neutron                        `https://opendev.org/openstack/neutron <https://opendev.org/openstack/neutron>`__
 openstack/neutron-dynamic-routing        `https://opendev.org/openstack/neutron-dynamic-routing <https://opendev.org/openstack/neutron-dynamic-routing>`__
@@ -78,20 +71,17 @@ openstack/neutron-fwaas-dashboard        `https://opendev.org/openstack/neutron-
 openstack/neutron-tempest-plugin         `https://opendev.org/openstack/neutron-tempest-plugin <https://opendev.org/openstack/neutron-tempest-plugin>`__
 openstack/neutron-vpnaas                 `https://opendev.org/openstack/neutron-vpnaas <https://opendev.org/openstack/neutron-vpnaas>`__
 openstack/neutron-vpnaas-dashboard       `https://opendev.org/openstack/neutron-vpnaas-dashboard <https://opendev.org/openstack/neutron-vpnaas-dashboard>`__
-openstack/nova-powervm                   `https://opendev.org/openstack/nova-powervm <https://opendev.org/openstack/nova-powervm>`__
+openstack/nova                           `https://opendev.org/openstack/nova <https://opendev.org/openstack/nova>`__
 openstack/octavia                        `https://opendev.org/openstack/octavia <https://opendev.org/openstack/octavia>`__
 openstack/octavia-dashboard              `https://opendev.org/openstack/octavia-dashboard <https://opendev.org/openstack/octavia-dashboard>`__
 openstack/octavia-tempest-plugin         `https://opendev.org/openstack/octavia-tempest-plugin <https://opendev.org/openstack/octavia-tempest-plugin>`__
 openstack/openstacksdk                   `https://opendev.org/openstack/openstacksdk <https://opendev.org/openstack/openstacksdk>`__
 openstack/osprofiler                     `https://opendev.org/openstack/osprofiler <https://opendev.org/openstack/osprofiler>`__
+openstack/ovn-bgp-agent                  `https://opendev.org/openstack/ovn-bgp-agent <https://opendev.org/openstack/ovn-bgp-agent>`__
 openstack/ovn-octavia-provider           `https://opendev.org/openstack/ovn-octavia-provider <https://opendev.org/openstack/ovn-octavia-provider>`__
 openstack/rally-openstack                `https://opendev.org/openstack/rally-openstack <https://opendev.org/openstack/rally-openstack>`__
-openstack/sahara                         `https://opendev.org/openstack/sahara <https://opendev.org/openstack/sahara>`__
-openstack/sahara-dashboard               `https://opendev.org/openstack/sahara-dashboard <https://opendev.org/openstack/sahara-dashboard>`__
-openstack/senlin                         `https://opendev.org/openstack/senlin <https://opendev.org/openstack/senlin>`__
 openstack/shade                          `https://opendev.org/openstack/shade <https://opendev.org/openstack/shade>`__
 openstack/skyline-apiserver              `https://opendev.org/openstack/skyline-apiserver <https://opendev.org/openstack/skyline-apiserver>`__
-openstack/solum                          `https://opendev.org/openstack/solum <https://opendev.org/openstack/solum>`__
 openstack/storlets                       `https://opendev.org/openstack/storlets <https://opendev.org/openstack/storlets>`__
 openstack/tacker                         `https://opendev.org/openstack/tacker <https://opendev.org/openstack/tacker>`__
 openstack/tap-as-a-service               `https://opendev.org/openstack/tap-as-a-service <https://opendev.org/openstack/tap-as-a-service>`__
@@ -184,6 +174,7 @@ x/trio2o                                 `https://opendev.org/x/trio2o <https://
 x/valet                                  `https://opendev.org/x/valet <https://opendev.org/x/valet>`__
 x/vmware-nsx                             `https://opendev.org/x/vmware-nsx <https://opendev.org/x/vmware-nsx>`__
 x/vmware-vspc                            `https://opendev.org/x/vmware-vspc <https://opendev.org/x/vmware-vspc>`__
+x/whitebox-neutron-tempest-plugin        `https://opendev.org/x/whitebox-neutron-tempest-plugin <https://opendev.org/x/whitebox-neutron-tempest-plugin>`__
 ======================================== ===


@@ -0,0 +1,25 @@
+=======
+Tempest
+=======
+
+`Tempest`_ is the OpenStack Integration test suite. It is installed by default
+and is used to provide integration testing for many of the OpenStack services.
+Just like DevStack itself, it is possible to extend Tempest with plugins. In
+fact, many Tempest plugin packages also include DevStack plugin to do things
+like pre-create required static resources.
+
+The `Tempest documentation <Tempest>`_ provides a thorough guide to using
+Tempest. However, if you simply wish to run the standard set of Tempest tests
+against an existing deployment, you can do the following:
+
+.. code-block:: shell
+
+    cd /opt/stack/tempest
+    /opt/stack/data/venv/bin/tempest run ...
+
+The above assumes you have installed DevStack in the default location
+(configured via the ``DEST`` configuration variable) and have enabled
+virtualenv-based installation in the standard location (configured via the
+``USE_VENV`` and ``VENV_DEST`` configuration variables, respectively).
+
+.. _Tempest: https://docs.openstack.org/tempest/latest/
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import os.path
+import json
+
+server_address = "/tmp/openstack.sock"
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+try:
+    sock.connect(server_address)
+except socket.error as msg:
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+
+    char = sock.recv(1)
+    if len(char) == 0:
+        print("Unexpected end of file", file=sys.stderr)
+        sys.exit(1)
+
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+        if len(char) == 0:
+            print("Unexpected end of file", file=sys.stderr)
+            sys.exit(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+try:
+    env = {}
+    passenv = ["CINDER_VERSION",
+               "OS_AUTH_URL",
+               "OS_NO_CACHE",
+               "OS_PASSWORD",
+               "OS_PROJECT_NAME",
+               "OS_REGION_NAME",
+               "OS_TENANT_NAME",
+               "OS_USERNAME",
+               "OS_VOLUME_API_VERSION",
+               "OS_CLOUD"]
+    for name in passenv:
+        if name in os.environ:
+            env[name] = os.environ[name]
+
+    cmd = {
+        "app": os.path.basename(sys.argv[0]),
+        "env": env,
+        "argv": sys.argv[1:]
+    }
+    try:
+        image_idx = sys.argv.index('image')
+        create_idx = sys.argv.index('create')
+        missing_file = image_idx < create_idx and \
+                not any(x.startswith('--file') for x in sys.argv)
+    except ValueError:
+        missing_file = False
+
+    if missing_file:
+        # This means we were called with an image create command, but were
+        # not provided a --file option. That likely means we're being passed
+        # the image data to stdin, which won't work because we do not proxy
+        # stdin to the server. So, we just reject the operation and ask the
+        # caller to provide the file with --file instead.
+        # We've already connected to the server, we need to send it some dummy
+        # data so it doesn't wait forever.
+        send(sock, {})
+        print('Image create without --file is not allowed in server mode',
+              file=sys.stderr)
+        sys.exit(1)
+    else:
+        send(sock, cmd)
+
+    doc = recv(sock)
+    if doc["stdout"] != b'':
+        print(doc["stdout"], end='')
+    if doc["stderr"] != b'':
+        print(doc["stderr"], file=sys.stderr)
+    sys.exit(doc["status"])
+finally:
+    sock.close()
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+import sys
+import os
+import json
+
+from openstackclient import shell as osc_shell
+from io import StringIO
+
+server_address = "/tmp/openstack.sock"
+
+try:
+    os.unlink(server_address)
+except OSError:
+    if os.path.exists(server_address):
+        raise
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+print('starting up on %s' % server_address, file=sys.stderr)
+sock.bind(server_address)
+
+# Listen for incoming connections
+sock.listen(1)
+
+def send(sock, doc):
+    jdoc = json.dumps(doc)
+    sock.send(b'%d\n' % len(jdoc))
+    sock.sendall(jdoc.encode('utf-8'))
+
+def recv(sock):
+    length_str = b''
+    char = sock.recv(1)
+    while char != b'\n':
+        length_str += char
+        char = sock.recv(1)
+
+    total = int(length_str)
+
+    # use a memoryview to receive the data chunk by chunk efficiently
+    jdoc = memoryview(bytearray(total))
+    next_offset = 0
+    while total - next_offset > 0:
+        recv_size = sock.recv_into(jdoc[next_offset:], total - next_offset)
+        next_offset += recv_size
+    try:
+        doc = json.loads(jdoc.tobytes())
+    except (TypeError, ValueError) as e:
+        raise Exception('Data received was not in JSON format')
+    return doc
+
+while True:
+    csock, client_address = sock.accept()
+    try:
+        doc = recv(csock)
+
+        print("%s %s" % (doc["app"], doc["argv"]), file=sys.stderr)
+        oldenv = {}
+        for name in doc["env"].keys():
+            oldenv[name] = os.environ.get(name, None)
+            os.environ[name] = doc["env"][name]
+
+        try:
+            old_stdout = sys.stdout
+            old_stderr = sys.stderr
+            my_stdout = sys.stdout = StringIO()
+            my_stderr = sys.stderr = StringIO()
+
+            class Exit(BaseException):
+                def __init__(self, status):
+                    self.status = status
+
+            def noexit(stat):
+                raise Exit(stat)
+
+            sys.exit = noexit
+
+            if doc["app"] == "openstack":
+                sh = osc_shell.OpenStackShell()
+                ret = sh.run(doc["argv"])
+            else:
+                print("Unknown application %s" % doc["app"], file=sys.stderr)
+                ret = 1
+        except Exit as e:
+            ret = e.status
+        finally:
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+
+            for name in oldenv.keys():
+                if oldenv[name] is None:
+                    del os.environ[name]
+                else:
+                    os.environ[name] = oldenv[name]
+
+        send(csock, {
+            "stdout": my_stdout.getvalue(),
+            "stderr": my_stderr.getvalue(),
+            "status": ret,
+        })
+
+    except BaseException as e:
+        print(e, file=sys.stderr)
+    finally:
+        csock.close()
@@ -118,7 +118,7 @@ function _upload_image {
        useimport="--import"
    fi

-    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties < "${image}"
+    openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format "$container" --disk-format "$disk" $useimport $properties --file $(readlink -f "${image}")
 }

 # Retrieve an image from a URL and upload into Glance.
@@ -133,17 +133,28 @@ function upload_image {

    local image image_fname image_name

+    local max_attempts=5
+
    # Create a directory for the downloaded image tarballs.
    mkdir -p $FILES/images
    image_fname=`basename "$image_url"`
    if [[ $image_url != file* ]]; then
        # Downloads the image (uec ami+akistyle), then extracts it.
        if [[ ! -f $FILES/$image_fname || "$(stat -c "%s" $FILES/$image_fname)" = "0" ]]; then
-            wget --progress=dot:giga -c $image_url -O $FILES/$image_fname
-            if [[ $? -ne 0 ]]; then
-                echo "Not found: $image_url"
-                return
-            fi
+            for attempt in `seq $max_attempts`; do
+                local rc=0
+                wget --progress=dot:giga -c $image_url -O $FILES/$image_fname || rc=$?
+                if [[ $rc -ne 0 ]]; then
+                    if [[ "$attempt" -eq "$max_attempts" ]]; then
+                        echo "Not found: $image_url"
+                        return
+                    fi
+                    echo "Download failed, retrying in $attempt second, attempt: $attempt"
+                    sleep $attempt
+                else
+                    break
+                fi
+            done
        fi
        image="$FILES/${image_fname}"
    else
@@ -414,10 +425,10 @@ function upload_image {
        # kernel for use when uploading the root filesystem.
        local kernel_id="" ramdisk_id="";
        if [ -n "$kernel" ]; then
-            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki < "$kernel" -f value -c id)
+            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $(_image_properties_to_arg $img_property) --public --container-format aki --disk-format aki --file $(readlink -f "$kernel") -f value -c id)
        fi
        if [ -n "$ramdisk" ]; then
-            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari < "$ramdisk" -f value -c id)
+            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $(_image_properties_to_arg $img_property) --public --container-format ari --disk-format ari --file $(readlink -f "$ramdisk") -f value -c id)
        fi
        _upload_image "${image_name%.img}" ami ami "$image" ${kernel_id:+ kernel_id=$kernel_id} ${ramdisk_id:+ ramdisk_id=$ramdisk_id} $img_property
    fi
@@ -683,6 +694,8 @@ function setup_colorized_logging {
    iniset $conf_file DEFAULT logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_debug_format_suffix "[00;33mfrom (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s [01;35m%(instance)s[00m"
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_systemd_logging {
@@ -704,6 +717,9 @@ function setup_systemd_logging {
    iniset $conf_file DEFAULT logging_context_format_string "%(color)s%(levelname)s %(name)s [[01;36m%(global_request_id)s %(request_id)s [00;36m%(project_name)s %(user_name)s%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_default_format_string "%(color)s%(levelname)s %(name)s [[00;36m-%(color)s] [01;35m%(instance)s%(color)s%(message)s[00m"
    iniset $conf_file DEFAULT logging_exception_prefix "ERROR %(name)s [01;35m%(instance)s[00m"
+
+    # Enable or disable color for oslo.log
+    iniset $conf_file DEFAULT log_color $LOG_COLOR
 }

 function setup_standard_logging_identity {
@@ -236,6 +236,27 @@ function trueorfalse {
    $xtrace
 }

+# bool_to_int <True|False>
+#
+# Convert True|False to int 1 or 0
+# This function can be used to convert the output of trueorfalse
+# to an int follow c conventions where false is 0 and 1 it true.
+function bool_to_int {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    if [ -z $1 ]; then
+        die $LINENO "Bool value required"
+    fi
+    if [[ $1 == "True" ]] ; then
+        echo '1'
+    else
+        echo '0'
+    fi
+    $xtrace
+}
+
+
 function isset {
    [[ -v "$1" ]]
 }
@@ -380,9 +401,9 @@ function warn {
 # such as "install_package" further abstract things in better ways.
 #
 # ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
-# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
+# ``os_RELEASE`` - major release: ``22.04`` (Ubuntu), ``23`` (Fedora)
 # ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
-# ``os_CODENAME`` - vendor's codename for release: ``xenial``
+# ``os_CODENAME`` - vendor's codename for release: ``jammy``

 declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME

@@ -750,7 +771,7 @@ function get_default_host_ip {
    if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
        host_ip=""
        # Find the interface used for the default route
-        host_ip_iface=${host_ip_iface:-$(ip -f $af route | awk '/default/ {print $5}' | head -1)}
+        host_ip_iface=${host_ip_iface:-$(ip -f $af route list match default table all | grep via | awk '/default/ {print $5}' | head -1)}
        local host_ips
        host_ips=$(LC_ALL=C ip -f $af addr show ${host_ip_iface} | sed /temporary/d |awk /$af'/ {split($2,parts,"/");  print parts[1]}')
        local ip
@@ -2417,6 +2438,11 @@ function time_stop {
    _TIME_TOTAL[$name]=$(($total + $elapsed_time))
 }

+function install_openstack_cli_server {
+    export PATH=$TOP_DIR/files/openstack-cli-server:$PATH
+    run_process openstack-cli-server "$PYTHON $TOP_DIR/files/openstack-cli-server/openstack-cli-server"
+}
+
 function oscwrap {
    local xtrace
    xtrace=$(set +o | grep xtrace)
@@ -41,7 +41,10 @@ function setup_devstack_virtualenv {
        # This package is currently installed via the distro and not
        # available on pypi.
        python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
-        pip_install -U pip
+        pip_install -U pip setuptools
+        #NOTE(rpittau): workaround for simplejson removal in osc
+        #  https://review.opendev.org/c/openstack/python-openstackclient/+/920001
+        pip_install -U simplejson
    fi
    if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
        export PATH="$DEVSTACK_VENV/bin:$PATH"
@@ -405,6 +408,9 @@ function _setup_package_with_constraints_edit {
        # source we are about to do.
        local name
        name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
+        if [ -z $name ]; then
+            name=$(awk '/^name =/ {gsub(/"/, "", $3); print $3}' $project_dir/pyproject.toml)
+        fi
        $REQUIREMENTS_DIR/.venv/bin/edit-constraints \
            $REQUIREMENTS_DIR/upper-constraints.txt -- $name
    fi
@@ -468,7 +474,10 @@ function setup_package {
    pip_install $flags "$project_dir$extras"
    # ensure that further actions can do things like setup.py sdist
    if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
-        safe_chown -R $STACK_USER $1/*.egg-info
+        # egg-info is not created when project have pyproject.toml
+        if [ -d $1/*.egg-info ]; then
+            safe_chown -R $STACK_USER $1/*.egg-info
+        fi
    fi
 }

@@ -237,13 +237,17 @@ function restart_apache_server {
    restart_service $APACHE_NAME
 }

+# write_uwsgi_config() - Create a new uWSGI config file
 function write_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
    local http=$4
-    local name=""
-    name=$(basename $wsgi)
+    local name=$5
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.
@@ -258,38 +262,46 @@ function write_uwsgi_config {
    local socket="$socket_dir/${name}.socket"

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
-    iniset "$file" uwsgi processes $API_WORKERS
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given. Note that the regex isn't exhaustive - neither Python modules
+    # nor Python variables can start with a number - but it's "good enough"
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,python3
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
+    iniset "$conf" uwsgi thunder-lock true
    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
+    iniset "$conf" uwsgi lazy-apps true

    # If we said bind directly to http, then do that and don't start the apache proxy
    if [[ -n "$http" ]]; then
-        iniset "$file" uwsgi http $http
+        iniset "$conf" uwsgi http $http
    else
        local apache_conf=""
        apache_conf=$(apache_site_config_for $name)
-        iniset "$file" uwsgi socket "$socket"
-        iniset "$file" uwsgi chmod-socket 666
+        iniset "$conf" uwsgi socket "$socket"
+        iniset "$conf" uwsgi chmod-socket 666
        echo "ProxyPass \"${url}\" \"unix:${socket}|uwsgi://uwsgi-uds-${name}\" retry=0 acquire=1 " | sudo tee -a $apache_conf
        enable_apache_site $name
        restart_apache_server
@@ -303,47 +315,58 @@ function write_uwsgi_config {
 # but that involves having apache buffer the request before sending it to
 # uwsgi.
 function write_local_uwsgi_http_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local url=$3
-    name=$(basename $wsgi)
+    local name=$4
+
+    if [ -z "$name" ]; then
+        name=$(basename $wsgi)
+    fi

    # create a home for the sockets; note don't use /tmp -- apache has
    # a private view of it on some platforms.

    # always cleanup given that we are using iniset here
-    rm -rf $file
-    iniset "$file" uwsgi wsgi-file "$wsgi"
+    rm -rf $conf
+    # Set either the module path or wsgi script path depending on what we've
+    # been given
+    if [[ "$wsgi" =~ ^[a-zA-Z0-9_.]+:[a-zA-Z0-9_]+$ ]]; then
+        iniset "$conf" uwsgi module "$wsgi"
+    else
+        deprecated 'Configuring uWSGI with a WSGI file is deprecated, use module paths instead'
+        iniset "$conf" uwsgi wsgi-file "$wsgi"
+    fi
    port=$(get_random_port)
-    iniset "$file" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
-    iniset "$file" uwsgi processes $API_WORKERS
+    iniset "$conf" uwsgi http-socket "$APACHE_LOCAL_HOST:$port"
+    iniset "$conf" uwsgi processes $API_WORKERS
    # This is running standalone
-    iniset "$file" uwsgi master true
+    iniset "$conf" uwsgi master true
    # Set die-on-term & exit-on-reload so that uwsgi shuts down
-    iniset "$file" uwsgi die-on-term true
-    iniset "$file" uwsgi exit-on-reload false
-    iniset "$file" uwsgi enable-threads true
-    iniset "$file" uwsgi plugins http,python3
-    # uwsgi recommends this to prevent thundering herd on accept.
-    iniset "$file" uwsgi thunder-lock true
-    # Set hook to trigger graceful shutdown on SIGTERM
-    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    iniset "$conf" uwsgi die-on-term true
+    iniset "$conf" uwsgi exit-on-reload false
    # Set worker-reload-mercy so that worker will not exit till the time
    # configured after graceful shutdown
-    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi worker-reload-mercy $WORKER_TIMEOUT
+    iniset "$conf" uwsgi enable-threads true
+    iniset "$conf" uwsgi plugins http,python3
+    # uwsgi recommends this to prevent thundering herd on accept.
+    iniset "$conf" uwsgi thunder-lock true
+    # Set hook to trigger graceful shutdown on SIGTERM
+    iniset "$conf" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
    # Override the default size for headers from the 4k default.
-    iniset "$file" uwsgi buffer-size 65535
+    iniset "$conf" uwsgi buffer-size 65535
    # Make sure the client doesn't try to re-use the connection.
-    iniset "$file" uwsgi add-header "Connection: close"
+    iniset "$conf" uwsgi add-header "Connection: close"
    # This ensures that file descriptors aren't shared between processes.
-    iniset "$file" uwsgi lazy-apps true
-    iniset "$file" uwsgi chmod-socket 666
-    iniset "$file" uwsgi http-raw-body true
-    iniset "$file" uwsgi http-chunked-input true
-    iniset "$file" uwsgi http-auto-chunked true
-    iniset "$file" uwsgi http-keepalive false
+    iniset "$conf" uwsgi lazy-apps true
+    iniset "$conf" uwsgi chmod-socket 666
+    iniset "$conf" uwsgi http-raw-body true
+    iniset "$conf" uwsgi http-chunked-input true
+    iniset "$conf" uwsgi http-auto-chunked true
+    iniset "$conf" uwsgi http-keepalive false
    # Increase socket timeout for slow chunked uploads
-    iniset "$file" uwsgi socket-timeout 30
+    iniset "$conf" uwsgi socket-timeout 30

    enable_apache_mod proxy
    enable_apache_mod proxy_http
@@ -376,12 +399,18 @@ function write_local_proxy_http_config {
 }

 function remove_uwsgi_config {
-    local file=$1
+    local conf=$1
    local wsgi=$2
    local name=""
+    # TODO(stephenfin): Remove this call when everyone is using module path
+    # configuration instead of file path configuration
    name=$(basename $wsgi)

-    rm -rf $file
+    if [[ "$wsgi" = /* ]]; then
+        deprecated "Passing a wsgi script to remove_uwsgi_config is deprecated, pass an application name instead"
+    fi
+
+    rm -rf $conf
    disable_apache_site $name
 }

@@ -88,6 +88,10 @@ CINDER_SERVICE_REPORT_INTERVAL=${CINDER_SERVICE_REPORT_INTERVAL:-120}
 # thin provisioning.
 CINDER_LVM_TYPE=${CINDER_LVM_TYPE:-auto}

+# ``CINDER_USE_SERVICE_TOKEN`` is a mode where service token is passed along with
+# user token while communicating to external REST APIs like Glance.
+CINDER_USE_SERVICE_TOKEN=$(trueorfalse True CINDER_USE_SERVICE_TOKEN)
+
 # Default backends
 # The backend format is type:name where type is one of the supported backend
 # types (lvm, nfs, etc) and name is the identifier used in the Cinder
@@ -183,6 +187,12 @@ fi
 # Environment variables to configure the image-volume cache
 CINDER_IMG_CACHE_ENABLED=${CINDER_IMG_CACHE_ENABLED:-True}

+# Environment variables to configure the optimized volume upload
+CINDER_UPLOAD_OPTIMIZED=${CINDER_UPLOAD_OPTIMIZED:-False}
+
+# Environment variables to configure the internal tenant during optimized volume upload
+CINDER_UPLOAD_INTERNAL_TENANT=${CINDER_UPLOAD_INTERNAL_TENANT:-False}
+
 # For limits, if left unset, it will use cinder defaults of 0 for unlimited
 CINDER_IMG_CACHE_SIZE_GB=${CINDER_IMG_CACHE_SIZE_GB:-}
 CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
@@ -192,6 +202,11 @@ CINDER_IMG_CACHE_SIZE_COUNT=${CINDER_IMG_CACHE_SIZE_COUNT:-}
 # enable the cache for all cinder backends.
 CINDER_CACHE_ENABLED_FOR_BACKENDS=${CINDER_CACHE_ENABLED_FOR_BACKENDS:-$CINDER_ENABLED_BACKENDS}

+# Configure which cinder backends will have optimized volume upload, this takes the same
+# form as the CINDER_ENABLED_BACKENDS config option. By default it will
+# enable the cache for all cinder backends.
+CINDER_UPLOAD_OPTIMIZED_BACKENDS=${CINDER_UPLOAD_OPTIMIZED_BACKENDS:-$CINDER_ENABLED_BACKENDS}
+
 # Flag to set the oslo_policy.enforce_scope. This is used to switch
 # the  Volume API policies to start checking the scope of token. by default,
 # this flag is False.
@@ -275,7 +290,7 @@ function cleanup_cinder {
    fi

    stop_process "c-api"
-    remove_uwsgi_config "$CINDER_UWSGI_CONF" "$CINDER_UWSGI"
+    remove_uwsgi_config "$CINDER_UWSGI_CONF" "cinder-wsgi"
 }

 # configure_cinder() - Set config files, create data dirs, etc
@@ -353,6 +368,14 @@ function configure_cinder {
            iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}
        fi
        configure_cinder_image_volume_cache
+
+        # The upload optimization uses Cinder's clone volume functionality to
+        # clone the Image-Volume from source volume hence can only be
+        # performed when glance is using cinder as it's backend.
+        if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then
+            # Configure optimized volume upload
+            configure_cinder_volume_upload
+        fi
    fi

    if is_service_enabled c-bak && [[ -n "$CINDER_BACKUP_DRIVER" ]]; then
@@ -426,6 +449,10 @@ function configure_cinder {
        iniset $CINDER_CONF oslo_policy enforce_scope false
        iniset $CINDER_CONF oslo_policy enforce_new_defaults false
    fi
+
+    if [ "$CINDER_USE_SERVICE_TOKEN" == "True" ]; then
+        init_cinder_service_user_conf
+    fi
 }

 # create_cinder_accounts() - Set up common required cinder accounts
@@ -449,32 +476,19 @@ function create_cinder_accounts {

        create_service_user "cinder" $extra_role

-        # block-storage is the official service type
-        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
-        if [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
+        local cinder_api_url
+        if [[ "$CINDER_USE_MOD_WSGI" == "False" ]]; then
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT"
        else
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume"
        fi

+        # block-storage is the official service type
+        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
+        get_or_create_endpoint \
+            "block-storage" \
+            "$REGION_NAME" \
+            "$cinder_api_url/v3"
        configure_cinder_internal_tenant
    fi
 }
@@ -729,6 +743,24 @@ function configure_cinder_image_volume_cache {
    done
 }

+function configure_cinder_volume_upload {
+    # Expect UPLOAD_VOLUME_OPTIMIZED_FOR_BACKENDS to be a list of backends
+    # similar to CINDER_ENABLED_BACKENDS with NAME:TYPE where NAME will
+    # be the backend specific configuration stanza in cinder.conf.
+    local be be_name
+    for be in ${CINDER_UPLOAD_OPTIMIZED_BACKENDS//,/ }; do
+        be_name=${be##*:}
+
+        iniset $CINDER_CONF $be_name image_upload_use_cinder_backend $CINDER_UPLOAD_OPTIMIZED
+        iniset $CINDER_CONF $be_name image_upload_use_internal_tenant $CINDER_UPLOAD_INTERNAL_TENANT
+    done
+}
+
+function init_cinder_service_user_conf {
+    configure_keystone_authtoken_middleware $CINDER_CONF cinder service_user
+    iniset $CINDER_CONF service_user send_service_user_token True
+    iniset $CINDER_CONF service_user auth_strategy keystone
+}

 # Restore xtrace
 $_XTRACE_CINDER
@@ -46,6 +46,10 @@ function recreate_database_postgresql {
    createdb -h $DATABASE_HOST -U$DATABASE_USER -l C -T template0 -E utf8 $db
 }

+function _exit_pg_init {
+    sudo cat /var/lib/pgsql/initdb_postgresql.log
+}
+
 function configure_database_postgresql {
    local pg_conf pg_dir pg_hba check_role version
    echo_summary "Configuring and starting PostgreSQL"
@@ -53,7 +57,9 @@ function configure_database_postgresql {
        pg_hba=/var/lib/pgsql/data/pg_hba.conf
        pg_conf=/var/lib/pgsql/data/postgresql.conf
        if ! sudo [ -e $pg_hba ]; then
+            trap _exit_pg_init EXIT
            sudo postgresql-setup initdb
+            trap - EXIT
        fi
    elif is_ubuntu; then
        version=`psql --version | cut -d ' ' -f3 | cut -d. -f1-2`
@@ -51,7 +51,7 @@ function start_etcd3 {
    fi
    cmd+=" --listen-client-urls http://$SERVICE_HOST:$ETCD_PORT"
    if [ "$ENABLE_DEBUG_LOG_LEVEL" == "True" ]; then
-        cmd+=" --debug"
+        cmd+=" --log-level=debug"
    fi

    local unitfile="$SYSTEMD_DIR/$ETCD_SYSTEMD_SERVICE"
@@ -75,6 +75,7 @@ GLANCE_MULTIPLE_FILE_STORES=${GLANCE_MULTIPLE_FILE_STORES:-fast}
 GLANCE_DEFAULT_BACKEND=${GLANCE_DEFAULT_BACKEND:-fast}

 GLANCE_CACHE_DIR=${GLANCE_CACHE_DIR:=$DATA_DIR/glance/cache}
+GLANCE_CACHE_DRIVER=${GLANCE_CACHE_DRIVER:-centralized_db}

 # Full Glance functionality requires running in standalone mode. If we are
 # not in uwsgi mode, then we are standalone, otherwise allow separate control.
@@ -103,6 +104,9 @@ GLANCE_ENABLE_QUOTAS=$(trueorfalse True GLANCE_ENABLE_QUOTAS)
 # For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
 GLANCE_ENFORCE_SCOPE=$(trueorfalse True GLANCE_ENFORCE_SCOPE)

+# Flag to disable image format inspection on upload
+GLANCE_ENFORCE_IMAGE_FORMAT=$(trueorfalse True GLANCE_ENFORCE_IMAGE_FORMAT)
+
 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
 GLANCE_API_CONF=$GLANCE_CONF_DIR/glance-api.conf
@@ -167,6 +171,7 @@ function cleanup_glance {
        # Cleanup reserved stores directories
        sudo rm -rf $GLANCE_STAGING_DIR $GLANCE_TASKS_DIR
    fi
+    remove_uwsgi_config "$GLANCE_UWSGI_CONF" "glance-wsgi-api"
 }

 # Set multiple cinder store related config options for each of the cinder store
@@ -329,6 +334,7 @@ function configure_glance {
    iniset $GLANCE_API_CONF database connection $dburl
    iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_API_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_API_CONF oslo_concurrency lock_path $GLANCE_LOCK_DIR
    iniset $GLANCE_API_CONF paste_deploy flavor keystone+cachemanagement
    configure_keystone_authtoken_middleware $GLANCE_API_CONF glance
@@ -340,6 +346,7 @@ function configure_glance {
    # Only use these if you know what you are doing!  See OSSN-0065
    iniset $GLANCE_API_CONF DEFAULT show_image_direct_url $GLANCE_SHOW_DIRECT_URL
    iniset $GLANCE_API_CONF DEFAULT show_multiple_locations $GLANCE_SHOW_MULTIPLE_LOCATIONS
+    iniset $GLANCE_API_CONF image_format require_image_format_match $GLANCE_ENFORCE_IMAGE_FORMAT

    # Configure glance_store
    configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES
@@ -392,6 +399,7 @@ function configure_glance {
    iniset $GLANCE_CACHE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
    iniset $GLANCE_CACHE_CONF DEFAULT use_syslog $SYSLOG
    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
+    iniset $GLANCE_CACHE_CONF DEFAULT image_cache_driver $GLANCE_CACHE_DRIVER
    iniset $GLANCE_CACHE_CONF DEFAULT auth_url $KEYSTONE_SERVICE_URI
    iniset $GLANCE_CACHE_CONF DEFAULT admin_tenant_name $SERVICE_PROJECT_NAME
    iniset $GLANCE_CACHE_CONF DEFAULT admin_user glance
@@ -429,6 +437,7 @@ function configure_glance {
        iniset $GLANCE_API_CONF DEFAULT bind_host $GLANCE_SERVICE_LISTEN_ADDRESS
        iniset $GLANCE_API_CONF DEFAULT bind_port $GLANCE_SERVICE_PORT_INT
        iniset $GLANCE_API_CONF DEFAULT workers "$API_WORKERS"
+        iniset $GLANCE_API_CONF DEFAULT worker_self_reference_url $GLANCE_URL
    fi

    if [[ "$GLANCE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
@@ -109,6 +109,10 @@ function configure_horizon {
        _horizon_config_set $local_settings "" OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT "True"
    fi

+    if is_service_enabled c-bak; then
+        _horizon_config_set $local_settings OPENSTACK_CINDER_FEATURES enable_backup "True"
+    fi
+
    # Create an empty directory that apache uses as docroot
    sudo mkdir -p $HORIZON_DIR/.blackhole

@@ -169,6 +173,10 @@ function install_horizon {
    # Apache installation, because we mark it NOPRIME
    install_apache_wsgi

+    # Install the memcache library so that horizon can use memcached as its
+    # cache backend
+    pip_install_gr pymemcache
+
    git_clone $HORIZON_REPO $HORIZON_DIR $HORIZON_BRANCH
 }

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Kernel Samepage Merging (KSM)
+# -----------------------------
+
+# Processes that mark their memory as mergeable can share identical memory
+# pages if KSM is enabled. This is particularly useful for nova + libvirt
+# backends but any other setup that marks its memory as mergeable can take
+# advantage. The drawback is there is higher cpu load; however, we tend to
+# be memory bound not cpu bound so enable KSM by default but allow people
+# to opt out if the CPU time is more important to them.
+ENABLE_KSM=$(trueorfalse True ENABLE_KSM)
+ENABLE_KSMTUNED=$(trueorfalse True ENABLE_KSMTUNED)
+function configure_ksm {
+    if [[ $ENABLE_KSMTUNED == "True" ]] ; then
+        install_package "ksmtuned"
+    fi
+    if [[ -f /sys/kernel/mm/ksm/run ]] ; then
+        echo $(bool_to_int ENABLE_KSM) | sudo tee /sys/kernel/mm/ksm/run
+    fi
+}
+
+# Compressed swap (ZSWAP)
+#------------------------
+
+# as noted in the kernel docs https://docs.kernel.org/admin-guide/mm/zswap.html
+# Zswap is a lightweight compressed cache for swap pages.
+# It takes pages that are in the process of being swapped out and attempts
+# to compress them into a dynamically allocated RAM-based memory pool.
+# zswap basically trades CPU cycles for potentially reduced swap I/O.
+# This trade-off can also result in a significant performance improvement
+# if reads from the compressed cache are faster than reads from a swap device.
+
+ENABLE_ZSWAP=$(trueorfalse False ENABLE_ZSWAP)
+# lz4 is very fast although it does not have the best compression
+# zstd has much better compression but more latency
+ZSWAP_COMPRESSOR=${ZSWAP_COMPRESSOR:="lz4"}
+ZSWAP_ZPOOL=${ZSWAP_ZPOOL:="z3fold"}
+function configure_zswap {
+    if [[ $ENABLE_ZSWAP == "True" ]] ; then
+        # Centos 9 stream seems to only support enabling but not run time
+        # tuning so dont try to choose better default on centos
+        if is_ubuntu; then
+            echo ${ZSWAP_COMPRESSOR} | sudo tee /sys/module/zswap/parameters/compressor
+            echo ${ZSWAP_ZPOOL} | sudo tee /sys/module/zswap/parameters/zpool
+        fi
+        echo 1 | sudo tee /sys/module/zswap/parameters/enabled
+        # print curent zswap kernel config
+        sudo grep -R . /sys/module/zswap/parameters || /bin/true
+    fi
+}
+
+ENABLE_SYSCTL_MEM_TUNING=$(trueorfalse False ENABLE_SYSCTL_MEM_TUNING)
+function configure_sysctl_mem_parmaters {
+    if [[ $ENABLE_SYSCTL_MEM_TUNING == "True" ]] ; then
+        # defer write when memory is available
+        sudo sysctl -w vm.dirty_ratio=60
+        sudo sysctl -w vm.dirty_background_ratio=10
+        sudo sysctl -w vm.vfs_cache_pressure=50
+        # assume swap is compressed so on new kernels
+        # give it equal priority as page cache which is
+        # uncompressed. on kernels < 5.8 the max is 100
+        # not 200 so it will strongly prefer swapping.
+        sudo sysctl -w vm.swappiness=100
+        sudo grep -R . /proc/sys/vm/  || /bin/true
+    fi
+}
+
+function configure_host_mem {
+    configure_zswap
+    configure_ksm
+    configure_sysctl_mem_parmaters
+}
+
+ENABLE_SYSCTL_NET_TUNING=$(trueorfalse False ENABLE_SYSCTL_NET_TUNING)
+function configure_sysctl_net_parmaters {
+    if [[ $ENABLE_SYSCTL_NET_TUNING == "True" ]] ; then
+        # detect dead TCP connections after 120 seconds
+        sudo sysctl -w net.ipv4.tcp_keepalive_time=60
+        sudo sysctl -w net.ipv4.tcp_keepalive_intvl=10
+        sudo sysctl -w net.ipv4.tcp_keepalive_probes=6
+        # reudce network latency for new connections
+        sudo sysctl -w net.ipv4.tcp_fastopen=3
+        # print tcp options
+        sudo grep -R . /proc/sys/net/ipv4/tcp* || /bin/true
+        # disable qos by default
+        sudo sysctl -w net.core.default_qdisc=pfifo_fast
+    fi
+}
+
+function configure_host_net {
+    configure_sysctl_net_parmaters
+}
+
+function tune_host {
+    configure_host_mem
+    configure_host_net
+}
@@ -31,7 +31,7 @@ function install_infra {
    local PIP_VIRTUAL_ENV="$REQUIREMENTS_DIR/.venv"
    [ ! -d $PIP_VIRTUAL_ENV ] && ${VIRTUALENV_CMD} $PIP_VIRTUAL_ENV
    # We don't care about testing git pbr in the requirements venv.
-    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr
+    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install -U pbr setuptools
    PIP_VIRTUAL_ENV=$PIP_VIRTUAL_ENV pip_install $REQUIREMENTS_DIR

    # Unset the PIP_VIRTUAL_ENV so that PBR does not end up trapped
@@ -150,7 +150,7 @@ function cleanup_keystone {
        sudo rm -f $(apache_site_config_for keystone)
    else
        stop_process "keystone"
-        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "$KEYSTONE_PUBLIC_UWSGI"
+        remove_uwsgi_config "$KEYSTONE_PUBLIC_UWSGI_CONF" "keystone-wsgi-public"
        sudo rm -f $(apache_site_config_for keystone-wsgi-public)
    fi
 }
@@ -88,12 +88,14 @@ export NEUTRON_TEST_CONFIG_FILE=${NEUTRON_TEST_CONFIG_FILE:-"$NEUTRON_CONF_DIR/d
 # enough
 NEUTRON_DEPLOY_MOD_WSGI=$(trueorfalse False NEUTRON_DEPLOY_MOD_WSGI)

+NEUTRON_UWSGI=neutron.wsgi.api:application
 NEUTRON_UWSGI_CONF=$NEUTRON_CONF_DIR/neutron-api-uwsgi.ini

 # If NEUTRON_ENFORCE_SCOPE == True, it will set "enforce_scope"
 # and "enforce_new_defaults" to True in the Neutron's config to enforce usage
-# of the new RBAC policies and scopes.
-NEUTRON_ENFORCE_SCOPE=$(trueorfalse False NEUTRON_ENFORCE_SCOPE)
+# of the new RBAC policies and scopes. Set it to False if you do not
+# want to run Neutron with new RBAC.
+NEUTRON_ENFORCE_SCOPE=$(trueorfalse True NEUTRON_ENFORCE_SCOPE)

 # Agent binaries.  Note, binary paths for other agents are set in per-service
 # scripts in lib/neutron_plugins/services/
@@ -157,6 +159,14 @@ if [[ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" && -z "$NEUTRON_ENDPOINT_SERVICE_NAME
    NEUTRON_ENDPOINT_SERVICE_NAME="networking"
 fi

+# Source install libraries
+ALEMBIC_REPO=${ALEMBIC_REPO:-https://github.com/sqlalchemy/alembic.git}
+ALEMBIC_DIR=${ALEMBIC_DIR:-$DEST/alembic}
+ALEMBIC_BRANCH=${ALEMBIC_BRANCH:-main}
+SQLALCHEMY_REPO=${SQLALCHEMY_REPO:-https://github.com/sqlalchemy/sqlalchemy.git}
+SQLALCHEMY_DIR=${SQLALCHEMY_DIR:-$DEST/sqlalchemy}
+SQLALCHEMY_BRANCH=${SQLALCHEMY_BRANCH:-main}
+
 # List of config file names in addition to the main plugin config file
 # To add additional plugin config files, use ``neutron_server_config_add``
 # utility function.  For example:
@@ -360,6 +370,24 @@ function _determine_config_l3 {
    echo "$opts"
 }

+function _enable_ovn_maintenance {
+    if [[ $Q_AGENT == "ovn" ]]; then
+        enable_service neutron-ovn-maintenance-worker
+    fi
+}
+
+function _run_ovn_maintenance {
+    if [[ $Q_AGENT == "ovn" ]]; then
+        run_process neutron-ovn-maintenance-worker "$NEUTRON_BIN_DIR/neutron-ovn-maintenance-worker $cfg_file_options"
+    fi
+}
+
+function _stop_ovn_maintenance {
+    if [[ $Q_AGENT == "ovn" ]]; then
+        stop_process neutron-ovn-maintenance-worker
+    fi
+}
+
 # For services and agents that require it, dynamically construct a list of
 # --config-file arguments that are passed to the binary.
 function determine_config_files {
@@ -439,7 +467,7 @@ function configure_neutron {
    iniset $NEUTRON_CONF DEFAULT rpc_state_report_workers 0

    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
-        write_uwsgi_config "$NEUTRON_UWSGI_CONF" "$NEUTRON_BIN_DIR/neutron-api" "/networking"
+        write_uwsgi_config "$NEUTRON_UWSGI_CONF" "$NEUTRON_UWSGI" "/networking" "" "neutron-api"
    fi
 }

@@ -458,7 +486,7 @@ function create_nova_conf_neutron {
    local conf=${1:-$NOVA_CONF}
    iniset $conf neutron auth_type "password"
    iniset $conf neutron auth_url "$KEYSTONE_SERVICE_URI"
-    iniset $conf neutron username "$Q_ADMIN_USERNAME"
+    iniset $conf neutron username nova
    iniset $conf neutron password "$SERVICE_PASSWORD"
    iniset $conf neutron user_domain_name "$SERVICE_DOMAIN_NAME"
    iniset $conf neutron project_name "$SERVICE_PROJECT_NAME"
@@ -524,6 +552,17 @@ function install_neutron {
        setup_dev_lib "neutron-lib"
    fi

+    # Install SQLAlchemy and alembic from git when these are required
+    # see https://bugs.launchpad.net/neutron/+bug/2042941
+    if use_library_from_git "sqlalchemy"; then
+        git_clone $SQLALCHEMY_REPO $SQLALCHEMY_DIR $SQLALCHEMY_BRANCH
+        setup_develop $SQLALCHEMY_DIR
+    fi
+    if use_library_from_git "alembic"; then
+        git_clone $ALEMBIC_REPO $ALEMBIC_DIR $ALEMBIC_BRANCH
+        setup_develop $ALEMBIC_DIR
+    fi
+
    git_clone $NEUTRON_REPO $NEUTRON_DIR $NEUTRON_BRANCH
    setup_develop $NEUTRON_DIR

@@ -608,13 +647,28 @@ function start_neutron_service_and_check {
        service_port=$Q_PORT_INT
        service_protocol="http"
    fi
+
    # Start the Neutron service
    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
+        # The default value of "rpc_workers" is None (not defined). If
+        # "rpc_workers" is explicitly set to 0, the RPC workers process
+        # should not be executed.
+        local rpc_workers
+        rpc_workers=$(iniget_multiline $NEUTRON_CONF DEFAULT rpc_workers)
+
        enable_service neutron-api
        run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF"
        neutron_url=$Q_PROTOCOL://$Q_HOST/
-        enable_service neutron-rpc-server
-        run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options"
+        if [ "$rpc_workers" != "0" ]; then
+            enable_service neutron-rpc-server
+        fi
+        enable_service neutron-periodic-workers
+        _enable_ovn_maintenance
+        if [ "$rpc_workers" != "0" ]; then
+            run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options"
+        fi
+        run_process neutron-periodic-workers "$NEUTRON_BIN_DIR/neutron-periodic-workers $cfg_file_options"
+        _run_ovn_maintenance
    else
        run_process q-svc "$NEUTRON_BIN_DIR/neutron-server $cfg_file_options"
        neutron_url=$service_protocol://$Q_HOST:$service_port/
@@ -686,7 +740,9 @@ function stop_other {

    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
        stop_process neutron-rpc-server
+        stop_process neutron-periodic-workers
        stop_process neutron-api
+        _stop_ovn_maintenance
    else
        stop_process q-svc
    fi
@@ -803,7 +859,9 @@ function cleanup_neutron {
    if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then
        stop_process neutron-api
        stop_process neutron-rpc-server
-        remove_uwsgi_config "$NEUTRON_UWSGI_CONF" "$NEUTRON_BIN_DIR/neutron-api"
+        stop_process neutron-periodic-workers
+        _stop_ovn_maintenance
+        remove_uwsgi_config "$NEUTRON_UWSGI_CONF" "neutron-api"
        sudo rm -f $(apache_site_config_for neutron-api)
    fi

@@ -1002,7 +1060,7 @@ function _configure_neutron_service {
    Q_API_PASTE_FILE=$NEUTRON_CONF_DIR/api-paste.ini
    cp $NEUTRON_DIR/etc/api-paste.ini $Q_API_PASTE_FILE

-    if [[ -n "$NEUTRON_ENDPOINT_SERVICE_NAME" ]]; then
+    if [[ "$NEUTRON_DEPLOY_MOD_WSGI" == "False" && -n "$NEUTRON_ENDPOINT_SERVICE_NAME" ]]; then
        _replace_api_paste_composite
    fi

@@ -91,9 +91,14 @@ OVN_GENEVE_OVERHEAD=${OVN_GENEVE_OVERHEAD:-38}
 # http://www.openvswitch.org/support/dist-docs/ovs-appctl.8.txt
 OVN_DBS_LOG_LEVEL=${OVN_DBS_LOG_LEVEL:-info}

+# OVN metadata agent configuration
 OVN_META_CONF=$NEUTRON_CONF_DIR/neutron_ovn_metadata_agent.ini
 OVN_META_DATA_HOST=${OVN_META_DATA_HOST:-$(ipv6_unquote $SERVICE_HOST)}

+# OVN agent configuration
+OVN_AGENT_CONF=$NEUTRON_CONF_DIR/plugins/ml2/ovn_agent.ini
+OVN_AGENT_EXTENSIONS=${OVN_AGENT_EXTENSIONS:-}
+
 # If True (default) the node will be considered a gateway node.
 ENABLE_CHASSIS_AS_GW=$(trueorfalse True ENABLE_CHASSIS_AS_GW)
 OVN_L3_CREATE_PUBLIC_NETWORK=$(trueorfalse True OVN_L3_CREATE_PUBLIC_NETWORK)
@@ -132,6 +137,7 @@ OVN_RUNDIR=$OVS_PREFIX/var/run/ovn

 NEUTRON_OVN_BIN_DIR=$(get_python_exec_prefix)
 NEUTRON_OVN_METADATA_BINARY="neutron-ovn-metadata-agent"
+NEUTRON_OVN_AGENT_BINARY="neutron-ovn-agent"

 STACK_GROUP="$( id --group --name "$STACK_USER" )"

@@ -334,8 +340,24 @@ function compile_ovn {
        ./boot.sh
    fi

+    # NOTE(mnaser): OVN requires that you build using the OVS from the
+    #               submodule.
+    #
+    #               https://github.com/ovn-org/ovn/blob/3fb397b63663297acbcbf794e1233951222ae5af/Documentation/intro/install/general.rst#bootstrapping
+    #               https://github.com/ovn-org/ovn/issues/128
+    git submodule update --init
+    pushd ovs
+    if [ ! -f configure ] ; then
+        ./boot.sh
+    fi
    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
-        ./configure --with-ovs-source=$DEST/$OVS_REPO_NAME $prefix $localstatedir
+        ./configure
+    fi
+    make -j$(($(nproc) + 1))
+    popd
+
+    if [ ! -f config.status ] || [ configure -nt config.status ] ; then
+        ./configure $prefix $localstatedir
    fi
    make -j$(($(nproc) + 1))
    sudo make install
@@ -487,6 +509,8 @@ function configure_ovn_plugin {

        if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
+        elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then
+            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=True
        else
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_metadata_enabled=False
        fi
@@ -508,6 +532,8 @@ function configure_ovn_plugin {
    if is_service_enabled n-api-meta ; then
        if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent; then
            iniset $NOVA_CONF neutron service_metadata_proxy True
+        elif is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]]; then
+            iniset $NOVA_CONF neutron service_metadata_proxy True
        fi
    fi
 }
@@ -539,29 +565,42 @@ function configure_ovn {
    fi

    # Metadata
-    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent && is_service_enabled ovn-controller; then
+    local sample_file=""
+    local config_file=""
+    if is_service_enabled q-ovn-agent neutron-ovn-agent && [[ "$OVN_AGENT_EXTENSIONS" =~ 'metadata' ]] && is_service_enabled ovn-controller; then
+        sample_file=$NEUTRON_DIR/etc/neutron/plugins/ml2/ovn_agent.ini.sample
+        config_file=$OVN_AGENT_CONF
+    elif is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent && is_service_enabled ovn-controller; then
+        sample_file=$NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample
+        config_file=$OVN_META_CONF
+    fi
+    if [ -n "$config_file" ]; then
        sudo install -d -o $STACK_USER $NEUTRON_CONF_DIR

        mkdir -p $NEUTRON_DIR/etc/neutron/plugins/ml2
        (cd $NEUTRON_DIR && exec ./tools/generate_config_file_samples.sh)

-        cp $NEUTRON_DIR/etc/neutron_ovn_metadata_agent.ini.sample $OVN_META_CONF
-        configure_root_helper_options $OVN_META_CONF
+        cp $sample_file $config_file
+        configure_root_helper_options $config_file

-        iniset $OVN_META_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
-        iniset $OVN_META_CONF DEFAULT nova_metadata_host $OVN_META_DATA_HOST
-        iniset $OVN_META_CONF DEFAULT metadata_workers $API_WORKERS
-        iniset $OVN_META_CONF DEFAULT state_path $DATA_DIR/neutron
-        iniset $OVN_META_CONF ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
-        iniset $OVN_META_CONF ovn ovn_sb_connection $OVN_SB_REMOTE
+        iniset $config_file DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
+        iniset $config_file DEFAULT nova_metadata_host $OVN_META_DATA_HOST
+        iniset $config_file DEFAULT metadata_workers $API_WORKERS
+        iniset $config_file DEFAULT state_path $DATA_DIR/neutron
+        iniset $config_file ovs ovsdb_connection tcp:$OVSDB_SERVER_LOCAL_HOST:6640
+        iniset $config_file ovn ovn_sb_connection $OVN_SB_REMOTE
        if is_service_enabled tls-proxy; then
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_ca_cert $INT_CA_DIR/ca-chain.pem
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_certificate $INT_CA_DIR/$DEVSTACK_CERT_NAME.crt
-            iniset $OVN_META_CONF ovn \
+            iniset $config_file ovn \
                ovn_sb_private_key $INT_CA_DIR/private/$DEVSTACK_CERT_NAME.key
        fi
+        if [[ $config_file == $OVN_AGENT_CONF ]]; then
+            iniset $config_file agent extensions $OVN_AGENT_EXTENSIONS
+            iniset $config_file ovn ovn_nb_connection $OVN_NB_REMOTE
+        fi
    fi
 }

@@ -684,6 +723,9 @@ function _start_ovn_services {
    if is_service_enabled q-ovn-metadata-agent neutron-ovn-metadata-agent ; then
        _start_process "devstack@q-ovn-metadata-agent.service"
    fi
+    if is_service_enabled q-ovn-agent neutron-ovn-agent ; then
+        _start_process "devstack@q-ovn-agent.service"
+    fi
 }

 # start_ovn() - Start running processes, including screen
@@ -750,6 +792,12 @@ function start_ovn {
        setup_logging $OVN_META_CONF
    fi

+    if is_service_enabled q-ovn-agent neutron-ovn-agent; then
+        run_process q-ovn-agent "$NEUTRON_OVN_BIN_DIR/$NEUTRON_OVN_AGENT_BINARY --config-file $OVN_AGENT_CONF"
+        # Format logging
+        setup_logging $OVN_AGENT_CONF
+    fi
+
    _start_ovn_services
 }

@@ -774,6 +822,12 @@ function stop_ovn {
        sudo pkill -9 -f "[h]aproxy" || :
        _stop_process "devstack@q-ovn-metadata-agent.service"
    fi
+    if is_service_enabled q-ovn-agent neutron-ovn-agent; then
+        # pkill takes care not to kill itself, but it may kill its parent
+        # sudo unless we use the "ps | grep [f]oo" trick
+        sudo pkill -9 -f "[h]aproxy" || :
+        _stop_process "devstack@q-ovn-agent.service"
+    fi
    if is_service_enabled ovn-controller-vtep ; then
        _stop_process "$OVN_CONTROLLER_VTEP_SERVICE"
    fi
@@ -53,11 +53,19 @@ NOVA_COND_CONF=$NOVA_CONF_DIR/nova.conf
 NOVA_CPU_CONF=$NOVA_CONF_DIR/nova-cpu.conf
 NOVA_FAKE_CONF=$NOVA_CONF_DIR/nova-fake.conf
 NOVA_API_DB=${NOVA_API_DB:-nova_api}
-NOVA_UWSGI=$NOVA_BIN_DIR/nova-api-wsgi
-NOVA_METADATA_UWSGI=$NOVA_BIN_DIR/nova-metadata-wsgi
+NOVA_UWSGI=nova.wsgi.osapi_compute:application
+NOVA_METADATA_UWSGI=nova.wsgi.metadata:application
 NOVA_UWSGI_CONF=$NOVA_CONF_DIR/nova-api-uwsgi.ini
 NOVA_METADATA_UWSGI_CONF=$NOVA_CONF_DIR/nova-metadata-uwsgi.ini

+# Allow forcing the stable compute uuid to something specific. This would be
+# done by deployment tools that pre-allocate the UUIDs, but it is also handy
+# for developers that need to re-stack a compute-only deployment multiple
+# times. Since the DB is non-local and not erased on an unstack, making it
+# stay the same each time is what developers want. Set to a uuid here or
+# leave it blank for default allocate-on-start behavior.
+NOVA_CPU_UUID=""
+
 # The total number of cells we expect. Must be greater than one and doesn't
 # count cell0.
 NOVA_NUM_CELLS=${NOVA_NUM_CELLS:-1}
@@ -165,6 +173,9 @@ NOVA_SHUTDOWN_TIMEOUT=${NOVA_SHUTDOWN_TIMEOUT:-0}
 # Whether to use Keystone unified limits instead of legacy quota limits.
 NOVA_USE_UNIFIED_LIMITS=$(trueorfalse False NOVA_USE_UNIFIED_LIMITS)

+# TB Cache Size in MiB for qemu guests
+NOVA_LIBVIRT_TB_CACHE_SIZE=${NOVA_LIBVIRT_TB_CACHE_SIZE:-0}
+
 # Functions
 # ---------

@@ -240,8 +251,8 @@ function cleanup_nova {

    stop_process "n-api"
    stop_process "n-api-meta"
-    remove_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI"
-    remove_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI"
+    remove_uwsgi_config "$NOVA_UWSGI_CONF" "nova-api"
+    remove_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "nova-metadata"

    if [[ "$NOVA_BACKEND" == "LVM" ]]; then
        clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME
@@ -541,11 +552,11 @@ function create_nova_conf {
    iniset $NOVA_CONF upgrade_levels compute "auto"

    if is_service_enabled n-api; then
-        write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute"
+        write_uwsgi_config "$NOVA_UWSGI_CONF" "$NOVA_UWSGI" "/compute" "" "nova-api"
    fi

    if is_service_enabled n-api-meta; then
-        write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}"
+        write_uwsgi_config "$NOVA_METADATA_UWSGI_CONF" "$NOVA_METADATA_UWSGI" "" "$SERVICE_LISTEN_ADDRESS:${METADATA_SERVICE_PORT}" "nova-metadata"
    fi

    if is_service_enabled ceilometer; then
@@ -629,7 +640,7 @@ function configure_placement_nova_compute {
    local conf=${1:-$NOVA_CONF}
    iniset $conf placement auth_type "password"
    iniset $conf placement auth_url "$KEYSTONE_SERVICE_URI"
-    iniset $conf placement username placement
+    iniset $conf placement username nova
    iniset $conf placement password "$SERVICE_PASSWORD"
    iniset $conf placement user_domain_name "$SERVICE_DOMAIN_NAME"
    iniset $conf placement project_name "$SERVICE_TENANT_NAME"
@@ -1046,7 +1057,7 @@ function start_nova_compute {

    # Set rebuild timeout longer for BFV instances because we likely have
    # slower disk than expected. Default is 20s/GB
-    iniset $NOVA_CPU_CONF DEFAULT reimage_timeout_per_gb 60
+    iniset $NOVA_CPU_CONF DEFAULT reimage_timeout_per_gb 180

    # Configure the OVSDB connection for os-vif
    if [ -n "$OVSDB_SERVER_LOCAL_HOST" ]; then
@@ -1058,7 +1069,14 @@ function start_nova_compute {
        iniset $NOVA_CPU_CONF workarounds libvirt_disable_apic True
    fi

+    if [[ "$NOVA_CPU_UUID" ]]; then
+        echo -n $NOVA_CPU_UUID > $NOVA_CONF_DIR/compute_id
+    fi
+
    if [[ "$VIRT_DRIVER" = 'libvirt' ]]; then
+        if [ ${NOVA_LIBVIRT_TB_CACHE_SIZE} -gt 0 ]; then
+            iniset $NOVA_CPU_CONF libvirt tb_cache_size ${NOVA_LIBVIRT_TB_CACHE_SIZE}
+        fi
        # The group **$LIBVIRT_GROUP** is added to the current user in this script.
        # ``sg`` is used in run_process to execute nova-compute as a member of the
        # **$LIBVIRT_GROUP** group.
@@ -56,6 +56,10 @@ function configure_nova_hypervisor {
    # arm64-specific configuration
    if is_arch "aarch64"; then
        iniset $NOVA_CONF libvirt cpu_mode "host-passthrough"
+        # NOTE(mnaser): We cannot have `cpu_model` set if the `cpu_mode` is
+        #               set to `host-passthrough`, or `nova-compute` refuses to
+        #               start.
+        inidelete $NOVA_CONF libvirt cpu_model
    fi

    if isset ENABLE_FILE_INJECTION; then
@@ -68,7 +68,7 @@ function is_placement_enabled {
 # runs that a clean run would need to clean up
 function cleanup_placement {
    sudo rm -f $(apache_site_config_for placement-api)
-    remove_uwsgi_config "$PLACEMENT_UWSGI_CONF" "$PLACEMENT_UWSGI"
+    remove_uwsgi_config "$PLACEMENT_UWSGI_CONF" "placement-api"
 }

 # _config_placement_apache_wsgi() - Set WSGI config files
@@ -844,14 +844,14 @@ function stop_swift {

 function swift_configure_tempurls {
    # note we are using swift credentials!
-    openstack --os-cloud "" \
-        --os-region-name $REGION_NAME \
-        --os-auth-url $KEYSTONE_SERVICE_URI \
-        --os-username=swift \
-        --os-password=$SERVICE_PASSWORD \
-        --os-user-domain-name=$SERVICE_DOMAIN_NAME \
-        --os-project-name=$SERVICE_PROJECT_NAME \
-        --os-project-domain-name=$SERVICE_DOMAIN_NAME \
+    openstack --os-cloud="" \
+        --os-region-name="$REGION_NAME" \
+        --os-auth-url="$KEYSTONE_SERVICE_URI" \
+        --os-username="swift" \
+        --os-password="$SERVICE_PASSWORD" \
+        --os-user-domain-name="$SERVICE_DOMAIN_NAME" \
+        --os-project-name="$SERVICE_PROJECT_NAME" \
+        --os-project-domain-name="$SERVICE_DOMAIN_NAME" \
        object store account \
        set --property "Temp-URL-Key=$SWIFT_TEMPURL_KEY"
 }
@@ -18,7 +18,7 @@
 #   - ``PUBLIC_NETWORK_NAME``
 #   - ``VIRT_DRIVER``
 #   - ``LIBVIRT_TYPE``
-#   - ``KEYSTONE_SERVICE_URI``, ``KEYSTONE_SERVICE_URI_V3`` from lib/keystone
+#   - ``KEYSTONE_SERVICE_URI_V3`` from lib/keystone
 #
 # Optional Dependencies:
 #
@@ -29,6 +29,7 @@
 # - ``DEFAULT_INSTANCE_USER``
 # - ``DEFAULT_INSTANCE_ALT_USER``
 # - ``CINDER_ENABLED_BACKENDS``
+# - ``CINDER_BACKUP_DRIVER``
 # - ``NOVA_ALLOW_DUPLICATE_NETWORKS``
 #
 # ``stack.sh`` calls the entry points in this order:
@@ -101,6 +102,9 @@ TEMPEST_USE_TEST_ACCOUNTS=$(trueorfalse False TEMPEST_USE_TEST_ACCOUNTS)
 # it will run tempest with
 TEMPEST_CONCURRENCY=${TEMPEST_CONCURRENCY:-$(nproc)}

+TEMPEST_FLAVOR_RAM=${TEMPEST_FLAVOR_RAM:-192}
+TEMPEST_FLAVOR_ALT_RAM=${TEMPEST_FLAVOR_ALT_RAM:-256}
+
 # Functions
 # ---------

@@ -294,13 +298,15 @@ function configure_tempest {
            if [[ ! ( $available_flavors =~ 'm1.nano' ) ]]; then
                # Determine the flavor disk size based on the image size.
                disk=$(image_size_in_gib $image_uuid)
-                openstack --os-cloud devstack-admin flavor create --id 42 --ram 128 --disk $disk --vcpus 1 --property hw_rng:allowed=True m1.nano
+                ram=${TEMPEST_FLAVOR_RAM}
+                openstack --os-cloud devstack-admin flavor create --id 42 --ram ${ram} --disk $disk --vcpus 1 --property hw_rng:allowed=True m1.nano
            fi
            flavor_ref=42
            if [[ ! ( $available_flavors =~ 'm1.micro' ) ]]; then
                # Determine the alt flavor disk size based on the alt image size.
                disk=$(image_size_in_gib $image_uuid_alt)
-                openstack --os-cloud devstack-admin flavor create --id 84 --ram 192 --disk $disk --vcpus 1 --property hw_rng:allowed=True m1.micro
+                ram=${TEMPEST_FLAVOR_ALT_RAM}
+                openstack --os-cloud devstack-admin flavor create --id 84 --ram ${ram} --disk $disk --vcpus 1 --property hw_rng:allowed=True m1.micro
            fi
            flavor_ref_alt=84
        else
@@ -351,6 +357,20 @@ function configure_tempest {
        fi
    fi

+    if is_service_enabled glance; then
+        git_clone $OSTESTIMAGES_REPO $OSTESTIMAGES_DIR $OSTESTIMAGES_BRANCH
+        pushd $OSTESTIMAGES_DIR
+        tox -egenerate
+        popd
+        iniset $TEMPEST_CONFIG image images_manifest_file ${OSTESTIMAGES_DIR}/images/manifest.yaml
+        local image_conversion
+        image_conversion=$(iniget $GLANCE_IMAGE_IMPORT_CONF image_conversion output_format)
+        if [[ -n "$image_conversion" ]]; then
+            iniset $TEMPEST_CONFIG image-feature-enabled image_conversion True
+        fi
+        iniset $TEMPEST_CONFIG image-feature-enabled image_format_enforcement $GLANCE_ENFORCE_IMAGE_FORMAT
+    fi
+
    iniset $TEMPEST_CONFIG network project_network_cidr $FIXED_RANGE

    ssh_connect_method=${TEMPEST_SSH_CONNECT_METHOD:-$ssh_connect_method}
@@ -380,7 +400,6 @@ function configure_tempest {
    iniset $TEMPEST_CONFIG volume build_timeout $BUILD_TIMEOUT

    # Identity
-    iniset $TEMPEST_CONFIG identity uri "$KEYSTONE_SERVICE_URI/v2.0/"
    iniset $TEMPEST_CONFIG identity uri_v3 "$KEYSTONE_SERVICE_URI_V3"
    iniset $TEMPEST_CONFIG identity user_lockout_failure_attempts $KEYSTONE_LOCKOUT_FAILURE_ATTEMPTS
    iniset $TEMPEST_CONFIG identity user_lockout_duration $KEYSTONE_LOCKOUT_DURATION
@@ -391,19 +410,7 @@ function configure_tempest {
        iniset $TEMPEST_CONFIG auth admin_project_name $admin_project_name
        iniset $TEMPEST_CONFIG auth admin_domain_name $admin_domain_name
    fi
-    if [ "$ENABLE_IDENTITY_V2" == "True" ]; then
-        # Run Identity API v2 tests ONLY if needed
-        iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 True
-    else
-        # Skip Identity API v2 tests by default
-        iniset $TEMPEST_CONFIG identity-feature-enabled api_v2 False
-    fi
    iniset $TEMPEST_CONFIG identity auth_version ${TEMPEST_AUTH_VERSION:-v3}
-    if [[ "$TEMPEST_AUTH_VERSION" != "v2" ]]; then
-        # we're going to disable v2 admin unless we're using v2 by default.
-        iniset $TEMPEST_CONFIG identity-feature-enabled api_v2_admin False
-    fi
-
    if is_service_enabled tls-proxy; then
        iniset $TEMPEST_CONFIG identity ca_certificates_file $SSL_BUNDLE_FILE
    fi
@@ -571,6 +578,9 @@ function configure_tempest {
        TEMPEST_VOLUME_REVERT_TO_SNAPSHOT=${TEMPEST_VOLUME_REVERT_TO_SNAPSHOT:-True}
    fi
    iniset $TEMPEST_CONFIG volume-feature-enabled volume_revert $(trueorfalse False TEMPEST_VOLUME_REVERT_TO_SNAPSHOT)
+    if [[ "$CINDER_BACKUP_DRIVER" == *"swift"* ]]; then
+        iniset $TEMPEST_CONFIG volume backup_driver swift
+    fi
    local tempest_volume_min_microversion=${TEMPEST_VOLUME_MIN_MICROVERSION:-None}
    local tempest_volume_max_microversion=${TEMPEST_VOLUME_MAX_MICROVERSION:-"latest"}
    if [ "$tempest_volume_min_microversion" == "None" ]; then
@@ -698,8 +708,6 @@ function configure_tempest {
    # test can be run with scoped token.
    if [[ "$KEYSTONE_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
        iniset $TEMPEST_CONFIG enforce_scope keystone true
-        iniset $TEMPEST_CONFIG auth admin_system 'all'
-        iniset $TEMPEST_CONFIG auth admin_project_name ''
    fi

    if [[ "$NOVA_ENFORCE_SCOPE" == True || "$ENFORCE_SCOPE" == True ]] ; then
@@ -7,9 +7,6 @@
 #   Set OS_USERNAME to override the default user name 'demo'
 #   Set ADMIN_PASSWORD to set the password for 'admin' and 'demo'

-# NOTE: support for the old NOVA_* novaclient environment variables has
-# been removed.
-
 if [[ -n "$1" ]]; then
    OS_USERNAME=$1
 fi
@@ -35,26 +32,11 @@ fi
 # Get some necessary configuration
 source $RC_DIR/lib/tls

-# The OpenStack ecosystem has standardized the term **project** as the
-# entity that owns resources.  In some places **tenant** remains
-# referenced, but in all cases this just means **project**.  We will
-# warn if we need to turn on legacy **tenant** support to have a
-# working environment.
+# Minimal configuration
+export OS_AUTH_TYPE=password
 export OS_PROJECT_NAME=${OS_PROJECT_NAME:-demo}
-
-echo "WARNING: setting legacy OS_TENANT_NAME to support cli tools."
-export OS_TENANT_NAME=$OS_PROJECT_NAME
-
-# In addition to the owning entity (project), nova stores the entity performing
-# the action as the **user**.
 export OS_USERNAME=${OS_USERNAME:-demo}
-
-# With Keystone you pass the keystone password instead of an api key.
-# Recent versions of novaclient use OS_PASSWORD instead of NOVA_API_KEYs
-# or NOVA_PASSWORD.
 export OS_PASSWORD=${ADMIN_PASSWORD:-secret}
-
-# Region
 export OS_REGION_NAME=${REGION_NAME:-RegionOne}

 # Set the host API endpoint. This will default to HOST_IP if SERVICE_IP_VERSION
@@ -73,30 +55,14 @@ else
    GLANCE_HOST=${GLANCE_HOST:-$HOST_IP}
 fi

-# Identity API version
-export OS_IDENTITY_API_VERSION=3
-
-# Ask keystoneauth1 to use keystone
-export OS_AUTH_TYPE=password
-
-# Authenticating against an OpenStack cloud using Keystone returns a **Token**
-# and **Service Catalog**.  The catalog contains the endpoints for all services
-# the user/project has access to - including nova, glance, keystone, swift, ...
-# We currently recommend using the version 3 *identity api*.
-#
-
 # If you don't have a working .stackenv, this is the backup position
 KEYSTONE_BACKUP=$SERVICE_PROTOCOL://$SERVICE_HOST:5000
 KEYSTONE_SERVICE_URI=${KEYSTONE_SERVICE_URI:-$KEYSTONE_BACKUP}

 export OS_AUTH_URL=${OS_AUTH_URL:-$KEYSTONE_SERVICE_URI}

-# Currently, in order to use openstackclient with Identity API v3,
-# we need to set the domain which the user and project belong to.
-if [ "$OS_IDENTITY_API_VERSION" = "3" ]; then
-    export OS_USER_DOMAIN_ID=${OS_USER_DOMAIN_ID:-"default"}
-    export OS_PROJECT_DOMAIN_ID=${OS_PROJECT_DOMAIN_ID:-"default"}
-fi
+export OS_USER_DOMAIN_ID=${OS_USER_DOMAIN_ID:-"default"}
+export OS_PROJECT_DOMAIN_ID=${OS_PROJECT_DOMAIN_ID:-"default"}

 # Set OS_CACERT to a default CA certificate chain if it exists.
 if [[ ! -v OS_CACERT ]] ; then
@@ -1,5 +0,0 @@
---
-fixes:
-  - |
-    Fixes a NotImplementedError when using the dbcounter SQLAlchemy plugin on
-    SQLAlchemy 2.x.
@@ -3,7 +3,9 @@
    executable: /bin/bash
    cmd: |
      source {{ devstack_conf_dir }}/stackrc
-      python3 {{ devstack_conf_dir }}/tools/get-stats.py \
+      source {{ devstack_conf_dir }}/inc/python
+      setup_devstack_virtualenv
+      $PYTHON {{ devstack_conf_dir }}/tools/get-stats.py \
        --db-user="$DATABASE_USER" \
        --db-pass="$DATABASE_PASSWORD" \
        --db-host="$DATABASE_HOST" \
@@ -43,9 +43,9 @@
      base_branch={{ devstack_sources_branch }}
      if git branch -a | grep "$base_branch" > /dev/null ; then
          git checkout $base_branch
-      elif [[ "$base_branch" == stable/* ]]; then
+      elif [[ "$base_branch" == stable/* ]] || [[ "$base_branch" == unmaintained/* ]]; then
          # Look for an eol tag for the stable branch.
-          eol_tag=${base_branch#stable/}-eol
+          eol_tag="${base_branch#*/}-eol"
          if git tag -l |grep $eol_tag >/dev/null; then
              git checkout $eol_tag
              git reset --hard $eol_tag
@@ -31,7 +31,7 @@ if is_service_enabled nova; then
    # ``demo``)

    # Get OpenStack user auth
-    source $TOP_DIR/openrc
+    export OS_CLOUD=devstack

    # Add first keypair found in localhost:$HOME/.ssh
    for i in $HOME/.ssh/id_rsa.pub $HOME/.ssh/id_dsa.pub; do
@@ -230,7 +230,7 @@ write_devstack_version

 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``FORCE=yes ./stack``
-SUPPORTED_DISTROS="bookworm|bullseye|jammy|rhel8|rhel9|openEuler-22.03"
+SUPPORTED_DISTROS="bookworm|jammy|noble|rhel9"

 if [[ ! ${DISTRO} =~ $SUPPORTED_DISTROS ]]; then
    echo "WARNING: this script has not been tested on $DISTRO"
@@ -302,23 +302,14 @@ function _install_epel {
 }

 function _install_rdo {
-    if [[ $DISTRO == "rhel8" ]]; then
+    if [[ $DISTRO == "rhel9" ]]; then
+        rdo_release=${TARGET_BRANCH#*/}
        if [[ "$TARGET_BRANCH" == "master" ]]; then
-            # rdo-release.el8.rpm points to latest RDO release, use that for master
-            sudo dnf -y install https://rdoproject.org/repos/rdo-release.el8.rpm
+            # adding delorean-deps repo to provide current master rpms
+            sudo wget https://trunk.rdoproject.org/centos9-master/delorean-deps.repo -O /etc/yum.repos.d/delorean-deps.repo
        else
-            # For stable branches use corresponding release rpm
-            rdo_release=$(echo $TARGET_BRANCH | sed "s|stable/||g")
-            sudo dnf -y install https://rdoproject.org/repos/openstack-${rdo_release}/rdo-release-${rdo_release}.el8.rpm
-        fi
-    elif [[ $DISTRO == "rhel9" ]]; then
-        if [[ "$TARGET_BRANCH" == "master" ]]; then
-            # rdo-release.el9.rpm points to latest RDO release, use that for master
-            sudo dnf -y install https://rdoproject.org/repos/rdo-release.el9.rpm
-        else
-            # For stable branches use corresponding release rpm
-            rdo_release=$(echo $TARGET_BRANCH | sed "s|stable/||g")
-            sudo dnf -y install https://rdoproject.org/repos/openstack-${rdo_release}/rdo-release-${rdo_release}.el9.rpm
+            # For stable/unmaintained branches use corresponding release rpm
+            sudo dnf -y install centos-release-openstack-${rdo_release}
        fi
    fi
    sudo dnf -y update
@@ -421,8 +412,12 @@ elif [[ $DISTRO == "openEuler-22.03" ]]; then
    # 1. the hostname package is not installed by default
    # 2. Some necessary packages are in openstack repo, for example liberasurecode-devel
    # 3. python3-pip can be uninstalled by `get_pip.py` automaticly.
-    install_package hostname openstack-release-wallaby
+    # 4. Ensure wget installation before use
+    install_package hostname openstack-release-wallaby wget
    uninstall_package python3-pip
+
+    # Add yum repository for libvirt7.X
+    sudo wget https://eur.openeuler.openatom.cn/coprs/g/sig-openstack/Libvirt-7.X/repo/openeuler-22.03_LTS/group_sig-openstack-Libvirt-7.X-openeuler-22.03_LTS.repo -O /etc/yum.repos.d/libvirt7.2.0.repo
 fi

 # Ensure python is installed
@@ -607,6 +602,12 @@ rm -f $SSL_BUNDLE_FILE
 source $TOP_DIR/lib/database
 source $TOP_DIR/lib/rpc_backend

+# load host tuning functions and defaults
+source $TOP_DIR/lib/host
+# tune host memory early to ensure zswap/ksm are configured before
+# doing memory intensive operation like cloning repos or unpacking packages.
+tune_host
+
 # Configure Projects
 # ==================

@@ -821,6 +822,7 @@ fixup_all

 if [[ "$GLOBAL_VENV" == "True" ]] ; then
    # TODO(frickler): find a better solution for this
+    sudo ln -sf /opt/stack/data/venv/bin/cinder-manage /usr/local/bin
    sudo ln -sf /opt/stack/data/venv/bin/cinder-rtstool /usr/local/bin
    sudo ln -sf /opt/stack/data/venv/bin/glance /usr/local/bin
    sudo ln -sf /opt/stack/data/venv/bin/nova-manage /usr/local/bin
@@ -1011,6 +1013,9 @@ if use_library_from_git "python-openstackclient"; then
    setup_dev_lib "python-openstackclient"
 else
    pip_install_gr python-openstackclient
+    if is_service_enabled openstack-cli-server; then
+        install_openstack_cli_server
+    fi
 fi

 # Installs alias for osc so that we can collect timing for all
@@ -1075,22 +1080,6 @@ fi
 # Save configuration values
 save_stackenv $LINENO

-# Kernel Samepage Merging (KSM)
-# -----------------------------
-
-# Processes that mark their memory as mergeable can share identical memory
-# pages if KSM is enabled. This is particularly useful for nova + libvirt
-# backends but any other setup that marks its memory as mergeable can take
-# advantage. The drawback is there is higher cpu load; however, we tend to
-# be memory bound not cpu bound so enable KSM by default but allow people
-# to opt out if the CPU time is more important to them.
-
-if [[ $ENABLE_KSM == "True" ]] ; then
-    if [[ -f /sys/kernel/mm/ksm/run ]] ; then
-        sudo sh -c "echo 1 > /sys/kernel/mm/ksm/run"
-    fi
-fi
-

 # Start Services
 # ==============
@@ -121,24 +121,11 @@ else
    SYSTEMCTL="sudo systemctl"
 fi

-
-# Whether or not to enable Kernel Samepage Merging (KSM) if available.
-# This allows programs that mark their memory as mergeable to share
-# memory pages if they are identical. This is particularly useful with
-# libvirt backends. This reduces memory usage at the cost of CPU overhead
-# to scan memory. We default to enabling it because we tend to be more
-# memory constrained than CPU bound.
-ENABLE_KSM=$(trueorfalse True ENABLE_KSM)
-
 # Passwords generated by interactive devstack runs
 if [[ -r $RC_DIR/.localrc.password ]]; then
    source $RC_DIR/.localrc.password
 fi

-# Control whether Python 3 should be used at all.
-# TODO(frickler): Drop this when all consumers are fixed
-export USE_PYTHON3=True
-
 # Adding the specific version of Python 3 to this variable will install
 # the app using that version of the interpreter instead of just 3.
 _DEFAULT_PYTHON3_VERSION="$(_get_python_version python3)"
@@ -175,10 +162,6 @@ else
    export PS4='+ $(short_source):   '
 fi

-# Configure Identity API version
-# TODO(frickler): Drop this when plugins no longer need it
-IDENTITY_API_VERSION=3
-
 # Global option for enforcing scope. If enabled, ENFORCE_SCOPE overrides
 # each services ${SERVICE}_ENFORCE_SCOPE variables
 ENFORCE_SCOPE=$(trueorfalse False ENFORCE_SCOPE)
@@ -207,8 +190,9 @@ fi
 USE_VENV=$(trueorfalse False USE_VENV)

 # Add packages that need to be installed into a venv but are not in any
-# requirmenets files here, in a comma-separated list
-ADDITIONAL_VENV_PACKAGES=${ADITIONAL_VENV_PACKAGES:-""}
+# requirements files here, in a comma-separated list.
+# Currently only used when USE_VENV is true (individual project venvs)
+ADDITIONAL_VENV_PACKAGES=${ADDITIONAL_VENV_PACKAGES:-""}

 # This can be used to turn database query logging on and off
 # (currently only implemented for MySQL backend)
@@ -265,7 +249,7 @@ REQUIREMENTS_DIR=${REQUIREMENTS_DIR:-$DEST/requirements}
 # Setting the variable to 'ALL' will activate the download for all
 # libraries.

-DEVSTACK_SERIES="2023.2"
+DEVSTACK_SERIES="2025.1"

 ##############
 #
@@ -320,6 +304,9 @@ TEMPEST_REPO=${TEMPEST_REPO:-${GIT_BASE}/openstack/tempest.git}
 TEMPEST_BRANCH=${TEMPEST_BRANCH:-$BRANCHLESS_TARGET_BRANCH}
 TEMPEST_VENV_UPPER_CONSTRAINTS=${TEMPEST_VENV_UPPER_CONSTRAINTS:-master}

+OSTESTIMAGES_REPO=${OSTESTIMAGES_REPO:-${GIT_BASE}/openstack/os-test-images.git}
+OSTESTIMAGES_BRANCH=${OSTESTIMAGES_BRANCH:-$BRANCHLESS_TARGET_BRANCH}
+OSTESTIMAGES_DIR=${DEST}/os-test-images

 ##############
 #
@@ -588,28 +575,6 @@ GITREPO["os-ken"]=${OS_KEN_REPO:-${GIT_BASE}/openstack/os-ken.git}
 GITBRANCH["os-ken"]=${OS_KEN_BRANCH:-$TARGET_BRANCH}
 GITDIR["os-ken"]=$DEST/os-ken

-##################
-#
-#  TripleO / Heat Agent Components
-#
-##################
-
-# run-parts script required by os-refresh-config
-DIB_UTILS_REPO=${DIB_UTILS_REPO:-${GIT_BASE}/openstack/dib-utils.git}
-DIB_UTILS_BRANCH=${DIB_UTILS_BRANCH:-$BRANCHLESS_TARGET_BRANCH}
-
-# os-apply-config configuration template tool
-OAC_REPO=${OAC_REPO:-${GIT_BASE}/openstack/os-apply-config.git}
-OAC_BRANCH=${OAC_BRANCH:-$TRAILING_TARGET_BRANCH}
-
-# os-collect-config configuration agent
-OCC_REPO=${OCC_REPO:-${GIT_BASE}/openstack/os-collect-config.git}
-OCC_BRANCH=${OCC_BRANCH:-$TRAILING_TARGET_BRANCH}
-
-# os-refresh-config configuration run-parts tool
-ORC_REPO=${ORC_REPO:-${GIT_BASE}/openstack/os-refresh-config.git}
-ORC_BRANCH=${ORC_BRANCH:-$TRAILING_TARGET_BRANCH}
-

 #################
 #
@@ -691,7 +656,7 @@ esac

 #IMAGE_URLS="https://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img" # cirros full disk image

-CIRROS_VERSION=${CIRROS_VERSION:-"0.6.2"}
+CIRROS_VERSION=${CIRROS_VERSION:-"0.6.3"}
 CIRROS_ARCH=${CIRROS_ARCH:-$(uname -m)}

 # Set default image based on ``VIRT_DRIVER`` and ``LIBVIRT_TYPE``, either of
@@ -708,11 +673,11 @@ if [[ "$DOWNLOAD_DEFAULT_IMAGES" == "True" ]]; then
                lxc) # the cirros root disk in the uec tarball is empty, so it will not work for lxc
                    DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-rootfs}
                    DEFAULT_IMAGE_FILE_NAME=${DEFAULT_IMAGE_FILE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-rootfs.img.gz}
-                    IMAGE_URLS+="https://download.cirros-cloud.net/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
+                    IMAGE_URLS+="https://github.com/cirros-dev/cirros/releases/download/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
                *) # otherwise, use the qcow image
                    DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk}
                    DEFAULT_IMAGE_FILE_NAME=${DEFAULT_IMAGE_FILE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img}
-                    IMAGE_URLS+="https://download.cirros-cloud.net/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
+                    IMAGE_URLS+="https://github.com/cirros-dev/cirros/releases/download/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
                esac
            ;;
        vsphere)
@@ -723,7 +688,7 @@ if [[ "$DOWNLOAD_DEFAULT_IMAGES" == "True" ]]; then
            # Use the same as the default for libvirt
            DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk}
            DEFAULT_IMAGE_FILE_NAME=${DEFAULT_IMAGE_FILE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img}
-            IMAGE_URLS+="http://download.cirros-cloud.net/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
+            IMAGE_URLS+="https://github.com/cirros-dev/cirros/releases/download/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
    esac
    DOWNLOAD_DEFAULT_IMAGES=False
 fi
@@ -73,8 +73,11 @@ logging.debug("Found %d projects" % len(projects))
 s = requests.Session()
 # sometimes gitea gives us a 500 error; retry sanely
 #  https://stackoverflow.com/a/35636367
+# We need to disable raise_on_status because if any repo endup with 500 then
+# propose-updates job which run this script will fail.
 retries = Retry(total=3, backoff_factor=1,
-                status_forcelist=[ 500 ])
+                status_forcelist=[ 500 ],
+                raise_on_status=False)
 s.mount('https://', HTTPAdapter(max_retries=retries))

 found_plugins = filter(functools.partial(has_devstack_plugin, s), projects)
@@ -76,7 +76,8 @@ def main():
        # with zuulv3 native jobs and ansible capture it may become
        # clearer what to do
        if HAS_DATE.search(line) is None:
-            now = datetime.datetime.utcnow()
+            now = datetime.datetime.now(datetime.timezone.utc).replace(
+                    tzinfo=None)
            ts_line = ("%s | %s" % (
                now.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3],
                line))
@@ -51,7 +51,7 @@ def get_options():


 def filename(dirname, name=""):
-    now = datetime.datetime.utcnow()
+    now = datetime.datetime.now(datetime.timezone.utc)
    fmt = "worlddump-%Y-%m-%d-%H%M%S"
    if name:
        fmt += "-" + name
@@ -12,7 +12,7 @@ basepython = python3
 # against devstack, just set BASHATE_INSTALL_PATH=/path/... to your
 # modified bashate tree
 deps =
-   {env:BASHATE_INSTALL_PATH:bashate==2.0.0}
+   {env:BASHATE_INSTALL_PATH:bashate}
 allowlist_externals = bash
 commands = bash -c "find {toxinidir}             \
         -not \( -type d -name .?\* -prune \)    \
@@ -168,6 +168,10 @@ if is_service_enabled etcd3; then
    cleanup_etcd3
 fi

+if is_service_enabled openstack-cli-server; then
+    stop_service devstack@openstack-cli-server
+fi
+
 stop_dstat

 # NOTE: Cinder automatically installs the lvm2 package, independently of the