Merge "Replace deprecated datetime.utcnow()"

pre-commit has been introduced to number of projects like oslo to run
lint checks such as hacking. The pre-commit config file does not affect
functionality, so devstack job is not needed when only the file is
updated.

Change-Id: I4294fe0c4df2c36c8575613b05a1f9c2eb745d18

.zuul.yaml

@@ -432,6 +432,8 @@
       - ^releasenotes/.*$
       # Translations
       - ^.*/locale/.*po$
+      # pre-commit config
+      - ^.pre-commit-config.yaml$
 
 - job:
     name: devstack-minimal

doc/requirements.txt

@@ -4,8 +4,4 @@ Pygments
 docutils
 sphinx>=2.0.0,!=2.1.0 # BSD
 openstackdocstheme>=2.2.1 # Apache-2.0
-nwdiag
-blockdiag
-sphinxcontrib-blockdiag
-sphinxcontrib-nwdiag
 zuul-sphinx>=0.2.0

doc/source/conf.py

@@ -23,14 +23,14 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinx.ext.autodoc',
-               'zuul_sphinx',
-               'openstackdocstheme',
-               'sphinxcontrib.blockdiag',
-               'sphinxcontrib.nwdiag' ]
+extensions = [
+    'sphinx.ext.autodoc',
+    'zuul_sphinx',
+    'openstackdocstheme',
+]
 
 # openstackdocstheme options
-openstackdocs_repo_name = 'openstack-dev/devstack'
+openstackdocs_repo_name = 'openstack/devstack'
 openstackdocs_pdf_link = True
 openstackdocs_bug_project = 'devstack'
 openstackdocs_bug_tag = ''

doc/source/guides/neutron.rst

@@ -41,19 +41,8 @@ network and is on a shared subnet with other machines.  The
 `local.conf` exhibited here assumes that 1500 is a reasonable MTU to
 use on that network.
 
-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                }
-        }
+.. image:: /assets/images/neutron-network-1.png
+   :alt: Network configuration for a single DevStack node
 
 
 DevStack Configuration
@@ -100,21 +89,8 @@ also want to do multinode testing and networking.
 Physical Network Setup
 ~~~~~~~~~~~~~~~~~~~~~~
 
-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network hardware_network {
-                        address = "172.18.161.0/24"
-                        router [ address = "172.18.161.1" ];
-                        devstack-1 [ address = "172.18.161.6" ];
-                        devstack-2 [ address = "172.18.161.7" ];
-                }
-        }
-
+.. image:: /assets/images/neutron-network-2.png
+   :alt: Network configuration for multiple DevStack nodes
 
 After DevStack installs and configures Neutron, traffic from guest VMs
 flows out of `devstack-2` (the compute node) and is encapsulated in a
@@ -222,8 +198,6 @@ connect OpenStack nodes (like `devstack-2`) together. This bridge is
 used so that project network traffic, using the VXLAN tunneling
 protocol, flows between each compute node where project instances run.
 
-
-
 DevStack Compute Configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -268,30 +242,8 @@ to the neutron L3 service.
 Physical Network Setup
 ----------------------
 
-.. nwdiag::
-
-        nwdiag {
-                inet [ shape = cloud ];
-                router;
-                inet -- router;
-
-                network provider_net {
-                        address = "203.0.113.0/24"
-                        router [ address = "203.0.113.1" ];
-                        controller;
-                        compute1;
-                        compute2;
-                }
-
-                network control_plane {
-                        router [ address = "10.0.0.1" ]
-                        address = "10.0.0.0/24"
-                        controller [ address = "10.0.0.2" ]
-                        compute1 [ address = "10.0.0.3" ]
-                        compute2 [ address = "10.0.0.4" ]
-                }
-        }
-
+.. image:: /assets/images/neutron-network-3.png
+   :alt: Network configuration for provider networks
 
 On a compute node, the first interface, eth0 is used for the OpenStack
 management (API, message bus, etc) as well as for ssh for an

doc/source/guides/nova.rst

@@ -122,7 +122,7 @@ when creating the server, for example:
 .. code-block:: shell
 
   $ openstack --os-compute-api-version 2.37 server create --flavor cirros256 \
-      --image cirros-0.6.2-x86_64-disk --nic none --wait test-server
+      --image cirros-0.6.3-x86_64-disk --nic none --wait test-server
 
 .. note:: ``--os-compute-api-version`` greater than or equal to 2.37 is
           required to use ``--nic=none``.

doc/source/index.rst

@@ -133,6 +133,8 @@ there.
 You can ``source openrc`` in your shell, and then use the
 ``openstack`` command line tool to manage your devstack.
 
+You can :ref:`create a VM and SSH into it <ssh>`.
+
 You can ``cd /opt/stack/tempest`` and run tempest tests that have
 been configured to work with your devstack.
 

doc/source/networking.rst

@@ -68,7 +68,7 @@ Shared Guest Interface
 .. warning::
 
    This is not a recommended configuration. Because of interactions
-   between ovs and bridging, if you reboot your box with active
+   between OVS and bridging, if you reboot your box with active
    networking you may lose network connectivity to your system.
 
 If you need your guests accessible on the network, but only have 1
@@ -114,3 +114,125 @@ For IPv6, ``FIXED_RANGE_V6`` will default to the first /64 of the value of
 ``FIXED_RANGE_V6`` will just use the value of that directly.
 ``SUBNETPOOL_PREFIX_V6`` will just default to the value of
 ``IPV6_ADDRS_SAFE_TO_USE`` directly.
+
+.. _ssh:
+
+SSH access to instances
+=======================
+
+To validate connectivity, you can create an instance using the
+``$PRIVATE_NETWORK_NAME`` network (default: ``private``), create a floating IP
+using the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``), and attach
+this floating IP to the instance:
+
+.. code-block:: shell
+
+    openstack keypair create --public-key ~/.ssh/id_rsa.pub test-keypair
+    openstack server create --network private --key-name test-keypair ... test-server
+    fip_id=$(openstack floating ip create public -f value -c id)
+    openstack server add floating ip test-server ${fip_id}
+
+Once done, ensure you have enabled SSH and ICMP (ping) access for the security
+group used for the instance. You can either create a custom security group and
+specify it when creating the instance or add it after creation, or you can
+modify the ``default`` security group created by default for each project.
+Let's do the latter:
+
+.. code-block:: shell
+
+    openstack security group rule create --proto icmp --dst-port 0 default
+    openstack security group rule create --proto tcp --dst-port 22 default
+
+Finally, SSH into the instance. If you used the Cirros instance uploaded by
+default, then you can run the following:
+
+.. code-block:: shell
+
+    openstack server ssh test-server -- -l cirros
+
+This will connect using the ``cirros`` user and the keypair you configured when
+creating the instance.
+
+Remote SSH access to instances
+==============================
+
+You can also SSH to created instances on your DevStack host from other hosts.
+This can be helpful if you are e.g. deploying DevStack in a VM on an existing
+cloud and wish to do development on your local machine. There are a few ways to
+do this.
+
+.. rubric:: Configure instances to be locally accessible
+
+The most obvious way is to configure guests to be locally accessible, as
+described `above <Locally Accessible Guests>`__. This has the advantage of
+requiring no further effort on the client. However, it is more involved and
+requires either support from your cloud or some inadvisable workarounds.
+
+.. rubric:: Use your DevStack host as a jump host
+
+You can choose to use your DevStack host as a jump host. To SSH to a instance
+this way, pass the standard ``-J`` option to the ``openstack ssh`` / ``ssh``
+command. For example:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -J username@devstack-host
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+This can also be configured via your ``~/.ssh/config`` file, making it rather
+effortless. However, it only allows SSH access. If you want to access e.g. a
+web application on the instance, you will need to configure an SSH tunnel and
+forward select ports using the ``-L`` option. For example, to forward HTTP
+traffic:
+
+.. code-block::
+
+    openstack server ssh test-server -- -l cirros -L 8080:username@devstack-host:80
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`, and ``username`` and ``devstack-host`` are the
+username and hostname of your DevStack host).
+
+As you can imagine, this can quickly get out of hand, particularly for more
+complex guest applications with multiple ports.
+
+.. rubric:: Use a proxy or VPN tool
+
+You can use a proxy or VPN tool to enable tunneling for the floating IP
+address range of the ``$PUBLIC_NETWORK_NAME`` network (default: ``public``)
+defined by ``$FLOATING_RANGE`` (default: ``172.24.4.0/24``). There are many
+such tools available to do this. For example, we could use a useful utility
+called `shuttle`__. To enable tunneling using ``shuttle``, first ensure you
+have allowed SSH and HTTP(S) traffic to your DevStack host. Allowing HTTP(S)
+traffic is necessary so you can use the OpenStack APIs remotely. How you do
+this will depend on where your DevStack host is running. Once this is done,
+install ``sshuttle`` on your localhost:
+
+.. code-block:: bash
+
+    sudo apt-get install sshuttle || yum install sshuttle
+
+Finally, start ``sshuttle`` on your localhost using the floating IP address
+range. For example, assuming you are using the default value for
+``$FLOATING_RANGE``, you can do:
+
+.. code-block:: bash
+
+    sshuttle -r username@devstack-host 172.24.4.0/24
+
+(where ``username`` and ``devstack-host`` are the username and hostname of your
+DevStack host).
+
+You should now be able to create an instance and SSH into it:
+
+.. code-block:: bash
+
+    openstack server ssh test-server -- -l cirros
+
+(where ``test-server`` is name of an existing instance, as described
+:ref:`previously <ssh>`)
+
+.. __: https://github.com/sshuttle/sshuttle

doc/source/plugin-registry.rst

@@ -48,9 +48,7 @@ openstack/ironic-inspector               `https://opendev.org/openstack/ironic-i
 openstack/ironic-prometheus-exporter     `https://opendev.org/openstack/ironic-prometheus-exporter <https://opendev.org/openstack/ironic-prometheus-exporter>`__
 openstack/ironic-ui                      `https://opendev.org/openstack/ironic-ui <https://opendev.org/openstack/ironic-ui>`__
 openstack/keystone                       `https://opendev.org/openstack/keystone <https://opendev.org/openstack/keystone>`__
-openstack/kuryr-kubernetes               `https://opendev.org/openstack/kuryr-kubernetes <https://opendev.org/openstack/kuryr-kubernetes>`__
 openstack/kuryr-libnetwork               `https://opendev.org/openstack/kuryr-libnetwork <https://opendev.org/openstack/kuryr-libnetwork>`__
-openstack/kuryr-tempest-plugin           `https://opendev.org/openstack/kuryr-tempest-plugin <https://opendev.org/openstack/kuryr-tempest-plugin>`__
 openstack/magnum                         `https://opendev.org/openstack/magnum <https://opendev.org/openstack/magnum>`__
 openstack/magnum-ui                      `https://opendev.org/openstack/magnum-ui <https://opendev.org/openstack/magnum-ui>`__
 openstack/manila                         `https://opendev.org/openstack/manila <https://opendev.org/openstack/manila>`__

lib/cinder

@@ -476,32 +476,19 @@ function create_cinder_accounts {
 
         create_service_user "cinder" $extra_role
 
-        # block-storage is the official service type
-        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
-        if [ "$CINDER_USE_MOD_WSGI" == "False" ]; then
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v3/\$(project_id)s"
+        local cinder_api_url
+        if [[ "$CINDER_USE_MOD_WSGI" == "False" ]]; then
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT"
         else
-            get_or_create_endpoint \
-                "block-storage" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
-
-            get_or_create_service "cinderv3" "volumev3" "Cinder Volume Service V3"
-            get_or_create_endpoint \
-                "volumev3" \
-                "$REGION_NAME" \
-                "$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume/v3/\$(project_id)s"
+            cinder_api_url="$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST/volume"
         fi
 
+        # block-storage is the official service type
+        get_or_create_service "cinder" "block-storage" "Cinder Volume Service"
+        get_or_create_endpoint \
+            "block-storage" \
+            "$REGION_NAME" \
+            "$cinder_api_url/v3"
         configure_cinder_internal_tenant
     fi
 }

lib/databases/postgresql

@@ -46,6 +46,10 @@ function recreate_database_postgresql {
     createdb -h $DATABASE_HOST -U$DATABASE_USER -l C -T template0 -E utf8 $db
 }
 
+function _exit_pg_init {
+    sudo cat /var/lib/pgsql/initdb_postgresql.log
+}
+
 function configure_database_postgresql {
     local pg_conf pg_dir pg_hba check_role version
     echo_summary "Configuring and starting PostgreSQL"
@@ -53,7 +57,9 @@ function configure_database_postgresql {
         pg_hba=/var/lib/pgsql/data/pg_hba.conf
         pg_conf=/var/lib/pgsql/data/postgresql.conf
         if ! sudo [ -e $pg_hba ]; then
+            trap _exit_pg_init EXIT
             sudo postgresql-setup initdb
+            trap - EXIT
         fi
     elif is_ubuntu; then
         version=`psql --version | cut -d ' ' -f3 | cut -d. -f1-2`

lib/glance

@@ -104,6 +104,9 @@ GLANCE_ENABLE_QUOTAS=$(trueorfalse True GLANCE_ENABLE_QUOTAS)
 # For more detail: https://docs.openstack.org/oslo.policy/latest/configuration/index.html#oslo_policy.enforce_scope
 GLANCE_ENFORCE_SCOPE=$(trueorfalse True GLANCE_ENFORCE_SCOPE)
 
+# Flag to disable image format inspection on upload
+GLANCE_ENFORCE_IMAGE_FORMAT=$(trueorfalse True GLANCE_ENFORCE_IMAGE_FORMAT)
+
 GLANCE_CONF_DIR=${GLANCE_CONF_DIR:-/etc/glance}
 GLANCE_METADEF_DIR=$GLANCE_CONF_DIR/metadefs
 GLANCE_API_CONF=$GLANCE_CONF_DIR/glance-api.conf
@@ -343,6 +346,7 @@ function configure_glance {
     # Only use these if you know what you are doing!  See OSSN-0065
     iniset $GLANCE_API_CONF DEFAULT show_image_direct_url $GLANCE_SHOW_DIRECT_URL
     iniset $GLANCE_API_CONF DEFAULT show_multiple_locations $GLANCE_SHOW_MULTIPLE_LOCATIONS
+    iniset $GLANCE_API_CONF image_format require_image_format_match $GLANCE_ENFORCE_IMAGE_FORMAT
 
     # Configure glance_store
     configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES

lib/swift

@@ -844,14 +844,14 @@ function stop_swift {
 
 function swift_configure_tempurls {
     # note we are using swift credentials!
-    openstack --os-cloud "" \
-        --os-region-name $REGION_NAME \
-        --os-auth-url $KEYSTONE_SERVICE_URI \
-        --os-username=swift \
-        --os-password=$SERVICE_PASSWORD \
-        --os-user-domain-name=$SERVICE_DOMAIN_NAME \
-        --os-project-name=$SERVICE_PROJECT_NAME \
-        --os-project-domain-name=$SERVICE_DOMAIN_NAME \
+    openstack --os-cloud="" \
+        --os-region-name="$REGION_NAME" \
+        --os-auth-url="$KEYSTONE_SERVICE_URI" \
+        --os-username="swift" \
+        --os-password="$SERVICE_PASSWORD" \
+        --os-user-domain-name="$SERVICE_DOMAIN_NAME" \
+        --os-project-name="$SERVICE_PROJECT_NAME" \
+        --os-project-domain-name="$SERVICE_DOMAIN_NAME" \
         object store account \
         set --property "Temp-URL-Key=$SWIFT_TEMPURL_KEY"
 }

lib/tempest

@@ -368,6 +368,7 @@ function configure_tempest {
         if [[ -n "$image_conversion" ]]; then
             iniset $TEMPEST_CONFIG image-feature-enabled image_conversion True
         fi
+        iniset $TEMPEST_CONFIG image-feature-enabled image_format_enforcement $GLANCE_ENFORCE_IMAGE_FORMAT
     fi
 
     iniset $TEMPEST_CONFIG network project_network_cidr $FIXED_RANGE

stackrc

@@ -249,7 +249,7 @@ REQUIREMENTS_DIR=${REQUIREMENTS_DIR:-$DEST/requirements}
 # Setting the variable to 'ALL' will activate the download for all
 # libraries.
 
-DEVSTACK_SERIES="2024.2"
+DEVSTACK_SERIES="2025.1"
 
 ##############
 #
@@ -656,7 +656,7 @@ esac
 
 #IMAGE_URLS="https://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img" # cirros full disk image
 
-CIRROS_VERSION=${CIRROS_VERSION:-"0.6.2"}
+CIRROS_VERSION=${CIRROS_VERSION:-"0.6.3"}
 CIRROS_ARCH=${CIRROS_ARCH:-$(uname -m)}
 
 # Set default image based on ``VIRT_DRIVER`` and ``LIBVIRT_TYPE``, either of
@@ -673,11 +673,11 @@ if [[ "$DOWNLOAD_DEFAULT_IMAGES" == "True" ]]; then
                 lxc) # the cirros root disk in the uec tarball is empty, so it will not work for lxc
                     DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-rootfs}
                     DEFAULT_IMAGE_FILE_NAME=${DEFAULT_IMAGE_FILE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-rootfs.img.gz}
-                    IMAGE_URLS+="https://download.cirros-cloud.net/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
+                    IMAGE_URLS+="https://github.com/cirros-dev/cirros/releases/download/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
                 *) # otherwise, use the qcow image
                     DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk}
                     DEFAULT_IMAGE_FILE_NAME=${DEFAULT_IMAGE_FILE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img}
-                    IMAGE_URLS+="https://download.cirros-cloud.net/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
+                    IMAGE_URLS+="https://github.com/cirros-dev/cirros/releases/download/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
                 esac
             ;;
         vsphere)
@@ -688,7 +688,7 @@ if [[ "$DOWNLOAD_DEFAULT_IMAGES" == "True" ]]; then
             # Use the same as the default for libvirt
             DEFAULT_IMAGE_NAME=${DEFAULT_IMAGE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk}
             DEFAULT_IMAGE_FILE_NAME=${DEFAULT_IMAGE_FILE_NAME:-cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img}
-            IMAGE_URLS+="http://download.cirros-cloud.net/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
+            IMAGE_URLS+="https://github.com/cirros-dev/cirros/releases/download/${CIRROS_VERSION}/${DEFAULT_IMAGE_FILE_NAME}";;
     esac
     DOWNLOAD_DEFAULT_IMAGES=False
 fi

tools/outfilter.py

@@ -76,7 +76,8 @@ def main():
         # with zuulv3 native jobs and ansible capture it may become
         # clearer what to do
         if HAS_DATE.search(line) is None:
-            now = datetime.datetime.utcnow()
+            now = datetime.datetime.now(datetime.timezone.utc).replace(
+                    tzinfo=None)
             ts_line = ("%s | %s" % (
                 now.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3],
                 line))