From 83eed99a9f36a2d3781c9fb3c432d349770053fd Mon Sep 17 00:00:00 2001 From: Balazs Gibizer Date: Thu, 26 Jun 2025 17:20:02 +0200 Subject: [PATCH] Run unit test with threading mode The py312-threading tox target will run the currently working unit tests with threading mode. We have an exclude list, those tests are failing or hanging. Also the current test list might still have unstable tests. This also adds a non voting zuul job to run the new target. Change-Id: Ibf41fede996fbf2ebaf6ae83df8cfde35acb2b7e Signed-off-by: Balazs Gibizer --- .zuul.yaml | 22 ++++++ threading_unit_test_excludes.txt | 123 +++++++++++++++++++++++++++++++ tools/generate-exclude.sh | 1 + tox.ini | 18 +++++ 4 files changed, 164 insertions(+) create mode 100644 threading_unit_test_excludes.txt create mode 100644 tools/generate-exclude.sh diff --git a/.zuul.yaml b/.zuul.yaml index 4c65a845f5be..2e494e0d24e6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -51,6 +51,26 @@ bindep_profile: test py312 timeout: 3600 +- job: + name: nova-tox-py312-threading + parent: openstack-tox-py312 + description: | + Run tox-based unit tests for the OpenStack Nova project + under cPython version 3.12 with eventlet disabled. + Uses tox with the ``py312-threading`` environment. + + required-projects: + # including nova here makes this job reusable by other projects and + # allow depends-on placement to work with our placement fixture + - openstack/nova + - openstack/placement + vars: + # explicitly stating the work dir makes this job reusable by other + # projects + zuul_work_dir: src/opendev.org/openstack/nova + bindep_profile: test py312 + tox_envlist: py312-threading + - job: name: nova-tox-validate-backport parent: openstack-tox @@ -821,6 +841,8 @@ - barbican-tempest-plugin-simple-crypto: irrelevant-files: *nova-base-irrelevant-files voting: false + - nova-tox-py312-threading: + voting: false gate: jobs: - nova-live-migration diff --git a/threading_unit_test_excludes.txt b/threading_unit_test_excludes.txt new file mode 100644 index 000000000000..83f2b6d082fa --- /dev/null +++ b/threading_unit_test_excludes.txt @@ -0,0 +1,123 @@ +nova.tests.unit.api.openstack.compute.test_availability_zone.ServersControllerCreateTestV21.test_create_instance_with_availability_zone +nova.tests.unit.api.openstack.compute.test_services.ServicesTestV253.test_update_disabled_no_reason_then_enable +nova.tests.unit.compute.test_shelve.ShelveComputeAPITestCase.test_unshelve_with_az_to_newaz +nova.tests.unit.compute.test_shelve.ShelveComputeAPITestCase.test_unshelve_without_az_to_newaz_and_host +nova.tests.unit.compute.test_shelve.ShelveComputeAPITestCase.test_unshelve_without_az_to_newaz +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_arq_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_destroy_build_request +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_exhaust_host_list +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_logs_selected_and_alts +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_no_host_mapping +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_reschedule_ignores_build_request +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_reschedule_not_recalc_mapping_if_claim_fails +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_reschedule_recalculates_provider_mapping +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_retry_exceeded +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_scheduler_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_scheduler_group_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_build_instances_update_instance_mapping +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_cold_migrate +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_cold_migrate_forced_shutdown +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_rebuild_instance_with_scheduler_group_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskAPITestCase.test_unshelve_offload_instance_on_host_with_request_spec +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_build_instances +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_build_instances_arq_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_build_instances_exhaust_host_list +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_build_instances_no_host_mapping +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_build_instances_reschedule_not_recalc_mapping_if_claim_fails +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_build_instances_scheduler_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_cold_migrate +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_rebuild_instance_with_scheduler_group_failure +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_unshelve_offload_instance_on_host_with_request_spec +nova.tests.unit.console.rfb.test_authvencrypt.RFBAuthSchemeVeNCryptTestCase.test_security_handshake_fails_on_ssl_failure +nova.tests.unit.console.rfb.test_authvencrypt.RFBAuthSchemeVeNCryptTestCase.test_security_handshake_without_x509 +nova.tests.unit.console.rfb.test_authvencrypt.RFBAuthSchemeVeNCryptTestCase.test_security_handshake_with_x509 +nova.tests.unit.objects.test_objects.TestObjectSerializer.test_object_tree_backport +nova.tests.unit.policies.test_availability_zone.AvailabilityZoneNoLegacyNoScopePolicyTest.test_availability_zone_detail_policy +nova.tests.unit.policies.test_availability_zone.AvailabilityZoneNoLegacyNoScopePolicyTest.test_availability_zone_list_policy +nova.tests.unit.policies.test_availability_zone.AvailabilityZonePolicyTest.test_availability_zone_detail_policy +nova.tests.unit.policies.test_availability_zone.AvailabilityZonePolicyTest.test_availability_zone_list_policy +nova.tests.unit.policies.test_availability_zone.AvailabilityZoneScopeTypePolicyTest.test_availability_zone_detail_policy +nova.tests.unit.policies.test_availability_zone.AvailabilityZoneScopeTypePolicyTest.test_availability_zone_list_policy +nova.tests.unit.policies.test_availability_zone.AZScopeTypeNoLegacyPolicyTest.test_availability_zone_list_policy +nova.tests.unit.storage.test_rbd.RbdTestCase.test_cleanup_volumes +nova.tests.unit.storage.test_rbd.RbdTestCase.test_cleanup_volumes_fail_not_found +nova.tests.unit.storage.test_rbd.RbdTestCase.test_cleanup_volumes_fail_other +nova.tests.unit.storage.test_rbd.RbdTestCase.test_cleanup_volumes_fail_snapshots +nova.tests.unit.storage.test_rbd.RbdTestCase.test_cleanup_volumes_pending_resize +nova.tests.unit.storage.test_rbd.RbdTestCase.test_cleanup_volumes_reverting_resize +nova.tests.unit.storage.test_rbd.RbdTestCase.test_destroy_volume +nova.tests.unit.test_availability_zones.AvailabilityZoneTestCases.test_get_availability_zones +nova.tests.unit.test_baserpc.BaseAPITestCase.test_get_backdoor_port +nova.tests.unit.test_baserpc.BaseAPITestCase.test_ping +nova.tests.unit.test_context.ContextTestCase.test_scatter_gather_cells_queued_task_cancelled +nova.tests.unit.test_rpc.TestRPC.test_get_server +nova.tests.unit.test_rpc.TestRPC.test_get_server_profiler_enabled +nova.tests.unit.test_utils.ExecutorStatsTestCase.test_stats_logged_eventlet +nova.tests.unit.test_utils.OsloServiceBackendSelectionTestCase.test_eventlet_selected +nova.tests.unit.virt.libvirt.test_driver.CacheConcurrencyTestCase.test_different_fname_concurrency +nova.tests.unit.virt.libvirt.test_driver.CacheConcurrencyTestCase.test_same_fname_concurrency +nova.tests.unit.virt.libvirt.test_driver.LibvirtDriverTestCase.test_rescue +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_image_cache_disk_reservation +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_for_pcpu_reshape +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_for_vgpu_reshape +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_for_vpmem +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_with_cpu_traits +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_with_file_backed_memory +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_with_tpm_traits +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_with_vgpus +nova.tests.unit.virt.libvirt.test_driver.TestUpdateProviderTree.test_update_provider_tree_zero_total +nova.tests.unit.virt.libvirt.volume.test_mount.HostMountStateTestCase.test_mount_concurrent +nova.tests.unit.virt.libvirt.volume.test_mount.HostMountStateTestCase.test_mount_concurrent_no_interfere +nova.tests.unit.virt.libvirt.volume.test_mount.MountManagerTestCase.test_host_up_waits_for_completion +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_attach_detach_different_power_states +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_attach_detach_volume +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_block_stats +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_destroy_instance +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_force_hard_reboot +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_console_output +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_diagnostics +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_info +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_instance_diagnostics +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_instance_disk_info +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_mks_console +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_serial_console +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_spice_console +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_get_vnc_console +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_live_migration +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_live_migration_force_complete +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_pause +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_poll_rebooting_instances +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_power_off +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_power_on_powered_off +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_power_on_running +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_reboot +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_rescue +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_restore_running +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_restore_soft_deleted +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_resume_state_on_host_boot +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_resume_suspended_instance +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_resume_unsuspended_instance +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_set_admin_password +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_snapshot_running +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_soft_delete +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_spawn +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_suspend +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_swap_volume +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_trigger_crash_dump +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_unpause_paused_instance +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_unpause_unpaused_instance +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_unplug_vifs_with_destroy_vifs_false +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_unplug_vifs_with_destroy_vifs_true +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_unrescue_rescued_instance +nova.tests.unit.virt.test_virt_drivers.LibvirtConnTestCase.test_unrescue_unrescued_instance +nova.tests.unit.virt.vmwareapi.test_vm_util.VMwareVMUtilTestCase.test_create_vm_invalid_guestid +# Independent failure ~10% with multiple possible error: +# - sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) not an error +# - sqlite3.OperationalError: cannot start a transaction within a transaction +# both triggered at: nova.compute.api.HostAPI._service_get_all_cells +nova.tests.unit.policies.test_availability_zone.AZScopeTypeNoLegacyPolicyTest.test_availability_zone_detail_policy +# Independent hang 100% +# after 160 seconds the test passes :/ +nova.tests.unit.conductor.test_conductor.ConductorTaskRPCAPITestCase.test_evacuate_old_rpc_without_target_state diff --git a/tools/generate-exclude.sh b/tools/generate-exclude.sh new file mode 100644 index 000000000000..7de931c4cea1 --- /dev/null +++ b/tools/generate-exclude.sh @@ -0,0 +1 @@ +grep -v "#" threading_unit_test_excludes.txt > /tmp/exclude.txt diff --git a/tox.ini b/tox.ini index 0908e8d8dd22..38d59798784b 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,24 @@ setenv = # by making greenlet leaks a failure. NOVA_RAISE_ON_GREENLET_LEAK=True +[testenv:py312-threading] +setenv = + {[testenv]setenv} +# we do not have any greenlet leaks in unit tests so enforce that +# by making greenlet leaks a failure. + NOVA_RAISE_ON_GREENLET_LEAK=True +# run the test without eventlet + OS_NOVA_DISABLE_EVENTLET_PATCHING=True + +commands = +# So far we have a list of failing test cases to filter out. Also note +# that there might be unstale tests. +# Our exclude list has comments we need to remove before it can be passed +# to stestr + bash tools/generate-exclude.sh + stestr run {posargs} --exclude-list /tmp/exclude.txt + stestr slowest + [testenv:functional{,-py310,-py311,-py312}] description = Run functional tests.